[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-1-en-105":3,"doc-seo-240338-105":53,"doc-detail-240338-en":127},{"code":4,"msg":5,"data":6},0,"success",[7,14,19,24,29,34,39,44,49],{"id":8,"doc_module":9,"doc_module_name":10,"category_name":11,"show_sort_weight":12,"slug":13},11,1,"Template","Presentations",90,"presentations",{"id":15,"doc_module":9,"doc_module_name":10,"category_name":16,"show_sort_weight":17,"slug":18},12,"Resumes",80,"resumes",{"id":20,"doc_module":9,"doc_module_name":10,"category_name":21,"show_sort_weight":22,"slug":23},14,"Invoices",70,"invoices",{"id":25,"doc_module":9,"doc_module_name":10,"category_name":26,"show_sort_weight":27,"slug":28},15,"Posters",60,"posters",{"id":30,"doc_module":9,"doc_module_name":10,"category_name":31,"show_sort_weight":32,"slug":33},16,"Social Media",50,"social-media",{"id":35,"doc_module":9,"doc_module_name":10,"category_name":36,"show_sort_weight":37,"slug":38},17,"Forms",40,"forms",{"id":40,"doc_module":9,"doc_module_name":10,"category_name":41,"show_sort_weight":42,"slug":43},18,"Letters",30,"letters",{"id":45,"doc_module":9,"doc_module_name":10,"category_name":46,"show_sort_weight":47,"slug":48},21,"Paper Templates",5,"papers-templates",{"id":50,"doc_module":9,"doc_module_name":10,"category_name":51,"show_sort_weight":4,"slug":52},158,"General","general-158",{"code":4,"msg":54,"data":55},"ok",{"site_id":56,"language":57,"slug":58,"title":59,"keywords":60,"description":61,"schema_data":62,"social_meta":120,"head_meta":122,"extra_data":124,"updated_unix":126},105,"en","cs224r-default-project-specification-rl-fine-tuning-of-language-models","CS224R Default Project Specification - RL Fine-Tuning of Language Models","","Explore reinforcement learning post-training for large language models through a default project built around the Countdown arithmetic reasoning task using the Qwen model. Implement supervised fine-tuning for warmstarting, pairwise preference optimization with a DPO/IPO-style objective, and online policy-gradient optimization with RLOO using a rule-based verifier reward. Build a rigorous evaluation setup comparing each stage to baselines, then design a chosen algorithmic extension to stress-test improvements on downstream performance.",{"@graph":63,"@context":119},[64,80,102],{"@type":65,"itemListElement":66},"BreadcrumbList",[67,71,74,77],{"item":68,"name":69,"@type":70,"position":9},"https://docshare.wps.com","Home","ListItem",{"item":72,"name":10,"@type":70,"position":73},"https://docshare.wps.com/template/",2,{"item":75,"name":11,"@type":70,"position":76},"https://docshare.wps.com/template/presentations/",3,{"item":78,"name":59,"@type":70,"position":79},"https://docshare.wps.com/template/cs224r-default-project-specification-rl-fine-tuning-of-language-models/240338/",4,{"url":78,"name":59,"@type":81,"image":82,"author":87,"headline":59,"publisher":90,"fileFormat":93,"inLanguage":57,"description":61,"dateModified":94,"datePublished":95,"encodingFormat":93,"isAccessibleForFree":96,"interactionStatistic":97},"DigitalDocument",{"url":83,"@type":84,"width":85,"height":86},"https://docshare.wps.com/thumbnails/cs224r-default-project-specification-rl-fine-tuning-of-language-models/240338.png","ImageObject",442,249,{"name":88,"@type":89},"Evangeline","Person",{"url":68,"name":91,"@type":92},"DocShare","Organization","application/pdf","2026-10-30","2026-09-11",true,{"@type":98,"interactionType":99,"userInteractionCount":101},"InteractionCounter",{"@type":100},"ViewAction",19,{"@type":103,"mainEntity":104},"FAQPage",[105,111,115],{"name":106,"@type":107,"acceptedAnswer":108},"What are the two main parts of the default project?","Question",{"text":109,"@type":110},"The project includes a core implementation section that implements parts of the RL stack for LLMs, and a research-oriented extension where you choose a research direction and evaluate its effect on downstream performance.","Answer",{"name":112,"@type":107,"acceptedAnswer":113},"What stages must be implemented in the default Countdown pipeline?",{"text":114,"@type":110},"You implement supervised fine-tuning (SFT), then pairwise preference optimization using a DPO/IPO-style objective, and finally online policy-gradient optimization with RLOO using a rule-based verifier reward.",{"name":116,"@type":107,"acceptedAnswer":117},"How should the results of different RL stages be evaluated?",{"text":118,"@type":110},"You construct an evaluation setup that rigorously compares the implemented stages against relevant baselines before proposing and testing your extension idea.","https://schema.org",{"og:url":78,"og:type":121,"og:title":59,"og:site_name":91,"og:description":61},"article",{"robots":123,"canonical":78},"index,follow",{"doc_id":125,"site_id":56},240338,1791743283,{"code":4,"msg":5,"data":128},{"doc_id":125,"user_id":129,"nickname":88,"user_avatar":130,"doc_module":9,"category_id":8,"category_name":11,"doc_title":59,"doc_description":61,"doc_content":131,"file_id":132,"file_url":133,"file_type":134,"file_size":135,"view_count":101,"is_deleted":4,"is_public":9,"is_downloadable":9,"audit_status":9,"page_count":136,"language":137,"language_code":57,"site_id":56,"html_lang":57,"table_of_contents":138,"faqs":139,"seo_title":140,"seo_description":61,"update_tm":141,"read_time":142},13056703019662,"https://ap-avatar.wpscdn.com/avatar/be000253a8e92610077?_k=1778726343310543188","| RL Fine-Tuning of Language Models\u003Cbr>CS 224R Default Project Specification |\n| --- |\n| You will explore how to use RL for post-training large language models (LLMs) through two main parts: (1) a core implementation section where you will get the chance to implement parts of the reinforcement learning stack for LLMs, and (2) a research-oriented extension where you explore a research direction of your choosing and evaluate whether it improves downstream performance.\u003Cbr>The default implementation is organized around the Countdown arithmetic reasoning task with the Qwen model. You will implement three core stages: (1) supervised fine-tuning (SFT) for warmstarting, (2) pairwise preference optimization using a DPO/IPO-style objective, and (3) online policy-gradient optimization with RLOO using a rule-based verifier reward. Next, you will construct an evaluation setup to rigorously compare these stages against relevant baselines. Finally, you will propose an extension of your choosing to explore how existing algorithmic ideas can be improved and stress-tested. |\n| \u003Cbr>Note on Clarifications\u003Cbr>Please use the default project Ed thread for questions and clarifications: [https://edstem.org/](https://edstem.org/)[ ](https://edstem.org/)[us/courses/97036/discussion/7926481](us/courses/97036/discussion/7926481. Course staff will use this thread to answer questions)[. Course staff will use this thread to answer questions](us/courses/97036/discussion/7926481. Course staff will use this thread to answer questions)[ ](us/courses/97036/discussion/7926481. Course staff will use this thread to answer questions)and share clarifications with students. |\n| \u003Cbr>\u003Cbr>Note on default project vs. custom project:\u003Cbr>It is not intended for the default final project to require less effort or work than the custom final project. The default project simply reduces the overall difficulty of devising your own project idea and evaluation methods, allowing students to commit an equivalent amount of effort to the provided problem. |\n|  |\n| \u003Cbr>\u003Cbr>Note on Default Project Tutorial\u003Cbr>We will have a virtual tutorial about the default project to walk through the details of the project and the starter code. Look out for announcements regarding this on Ed. |\n| \u003Cbr>\u003Cbr>Note on Implementation Integrity\u003Cbr>Please do not use/refer to code from any existing codebase, framework, or AI agent (e.g., Cursor, Codex, Claude Code, Antigravity, etc.) for ANY part of the default project except for the final extension. You may use required infrastructure libraries such as Hugging Face for model, tokenizer, and dataset loading, but you may not use high-level trainers such as SFTTrainer for the implementation component. The Stanford Honor Code applies! |\n| \u003Cbr>\u003Cbr>Note on OAE Extensions\u003Cbr>Assignment extensions granted by the Office of OAE do not apply to group assignments. However, if you are working individually they can apply. |\n| \u003Cbr>Note on Assistance\u003Cbr> TAs may not look at students’ code for either the default or custom final projects.  |\n|  |\n\nContents  \n1 Initial Implementation 4  \n1.1 Background ......................................... 4  \n1.2 Description of Tasks and Dataloading ........................... 5  \n1.2.1 Default Task: Math Reasoning with Countdown ................. 5  \n1.2.2 How Data Depends on the Training Algorithm .................. 5  \n1.2.3 Preference Datasets . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 5  \n1.2.4 Verifier-Based Datasets . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 6  \n1.2.5 Datasets Used in the Default Countdown Pipeline ................ 6  \n1.3 Method Implementation (Difficulty: Medium, Time: High) ............... 7  \n1.3.1 Supervised Fine-Tuning (SFT) ........................... 7  \n1.3.2 Preference Optimization (IPO) .......................... 7  \n1.3.3 Online Policy-Gradient (RLOO) .......................... 8  \n1.4 Evaluation Setup ...................................... 9  \n2 Extension to the RL ","cbCaih3BeQSCTlAq","https://ap.wps.com/l/cbCaih3BeQSCTlAq","pdf",496468,23,"English","# Initial Implementation\n## Background\n## Description of Tasks and Dataloading\n## Method Implementation\n## Evaluation Setup\n# Extension to the RL Methods Explored\n## Synthetic Data Augmentation & Pre-training Initialization\n## Improving Efficiency through Off-Policy Sampling\n## Incorporating Test Time Inference\n## Exploration in Discrete Token Spaces\n## Self-Play & Multi-Agent Co-evolution\n## Effective Tool-Integrated Reasoning\n## Learning with a Curriculum\n## Choose Your Own Adventure\n# Important Information\n## Project Mentors\n## Project Groups\n## Contributions\n## Honor Code and AI Tools Policy\n## Submission Format\n## Using Late Days\n## Computing Resources\n# Deliverables and Grading\n## Survey\n## Project Proposal + SFT Implementation\n## IPO + RLOO Milestone\n## Poster Presentation\n## Report\n# Expectation of the Default Project","[{\"question\":\"What are the two main parts of the default project?\",\"answer\":\"The project includes a core implementation section that implements parts of the RL stack for LLMs, and a research-oriented extension where you choose a research direction and evaluate its effect on downstream performance.\"},{\"question\":\"What stages must be implemented in the default Countdown pipeline?\",\"answer\":\"You implement supervised fine-tuning (SFT), then pairwise preference optimization using a DPO/IPO-style objective, and finally online policy-gradient optimization with RLOO using a rule-based verifier reward.\"},{\"question\":\"How should the results of different RL stages be evaluated?\",\"answer\":\"You construct an evaluation setup that rigorously compares the implemented stages against relevant baselines before proposing and testing your extension idea.\"}]","CS224R Default Project Specification - RL Fine-Tuning of Language Models | PDF",1789160153,8]