[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-detail-184331-en":3,"doc-seo-184331-105":30,"detail-sidebar-cat-0-en-105":92},{"code":4,"msg":5,"data":6},0,"success",{"doc_id":7,"user_id":8,"nickname":9,"user_avatar":10,"doc_module":4,"category_id":11,"category_name":12,"doc_title":13,"doc_description":14,"doc_content":15,"file_id":16,"file_url":17,"file_type":18,"file_size":19,"view_count":20,"is_deleted":4,"is_public":20,"is_downloadable":20,"audit_status":20,"page_count":21,"language":22,"language_code":23,"site_id":24,"html_lang":23,"table_of_contents":25,"faqs":26,"seo_title":27,"seo_description":14,"update_tm":28,"read_time":29},184331,687207024643,"Rhys","https://ap-avatar.wpscdn.com/davatar_3d24733baf745e90a7e4bdd5f77d97b2",8,"Research & Report","ICLR-2026-routing-manifold-alignment-improves-generalization-of-mixture-of-experts-llms-Paper-Conference - research results","A comparative evaluation table reports performance across multiple language and reasoning benchmarks for mixture-of-experts LLM variants. The results contrast baseline models with tuning strategies such as ICL, C3PO, Router Tuning, Oracle Tuning, Prefix Tuning, Prompt Tuning, and Dense BP. The method “RoMA (Ours)” shows consistent gains over baselines and several tuning baselines on datasets including MMLU, HellaSwag, ARC-C, ARC-E, PIQA, WinoGrande, BoolQ, and GSM8K, with an average score column summarizing improvements. Additional sections study effects under different active-parameter regimes and model sizes.","| Method | MMLU | Hella\u003Cbr>Swag | ARC-C | ARC-E | PIQA | Wino\u003Cbr>Grande | BoolQ | GSM8K | Avg |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| DeepSeekMoE-16B-A3B |  |  |  |  |  |  |  |  |  |\n| Base model | 46.2 | 78.0 | 50.3 | 73.8 | 79.9 | 70.1 | 72.3 | 62.2 | 66.6 |\n| Oracle | 63.8 | 92.5 | 70.8 | 85.2 | 90.3 | 82.1 | 83.2 | \u003Cbr>71.8 | 80.0 |\n| ICL | 49.0 | 81.6 | 56.3 | 76.2 | 81.4 | 72.3 | 75.8 | 65.7 | 69.8 |\n| C3PO | 55.4 | 85.7 | 61.6 | 80.7 | 85.8 | 77.5 | 78.2 | 68.5 | 74.2 |\n|  |  |  |  |  |  |  |  |  |  |\n| Router Tuning | 49.3 | 81.5 | 57.2 | 76.6 | 82.0 | 73.8 | 74.5 | 64.8 | 70.0 |\n| Oracle Tuning | 54.2 | 84.3 | 60.1 | 79.5 | 84.0 | 76.0 | 77.5 | 66.2 | 72.7 |\n| Prefix Tuning | 47.8 | 77.9 | 52.4 | 73.8 | 79.2 | 70.3 | 73.1 | 64.8 | 67.4 |\n| Prompt Tuning | 49.3 | 78.6 | 55.1 | 74.7 | 80.5 | 72.0 | 74.2 | 65.5 | 68.7 |\n| Dense BP | 50.1 | 80.2 | 54.8 | 77.3 | 81.7 | 74.2 | 76.1 | 63.9 | 69.8 |\n| RoMA (Ours) 56.8 87.9  61.4 81.5  85.2  76.8 80.6  67.4 74.7 |  |  |  |  |  |  |  |  |  |\n| OLMoE-7B-A1B |  |  |  |  |  |  |  |  |  |\n| Base model | 57.8 | 77.9 | 51.3 | 79.8 | 80.7 | 72.2 | 75.4 | 45.5 | 67.6 |\n| Oracle | 72.2 | 91.5 | 74.8 | 91.4 | 93.6 | 87.7 | 84.5 | \u003Cbr>53.2 | 81.1 |\n| ICL | 60.3 | 80.6 | 58.1 | 82.5 | 83.6 | 76.8 | 78.9 | 48.5 | 71.2 |\n| C3PO | 65.5 | 85.3 | 66.3 | 87.4 | 88.0 | 82.7 | 79.6 | 50.8 | 75.7 |\n|  |  |  |  |  |  |  |  |  |  |\n| Router Tuning | 63.2 | 81.7 | 62.5 | 83.8 | 80.9 | 75.3 | 77.8 | 47.2 | 71.6 |\n| Oracle Tuning | 66.8 | 84.2 | 65.4 | 86.1 | 86.2 | 80.5 | 79.9 | 49.0 | 74.8 |\n| Prefix Tuning | 59.3 | 78.2 | 54.5 | 80.4 | 82.1 | 73.5 | 76.8 | 46.7 | 68.9 |\n| Prompt Tuning | 59.7 | 79.5 | 55.9 | 81.3 | 82.4 | 74.1 | 77.2 | 47.3 | 69.7 |\n| Dense BP | 61.8 | 82.4 | 57.3 | 84.1 | 83.9 | 76.9 | 75.2 | 48.1 | 71.2 |\n| RoMA (Ours) 69.0 86.7 67.2 88.0 85.8  81.8 81.7  49.4 76.2 |  |  |  |  |  |  |  |  |  |\n| Qwen3-30B-A3B |  |  |  |  |  |  |  |  |  |\n| Base model | 74.2 | 68.5 | 56.8 | 84.3 | 78.5 | 65.2 | 81.3 | 83.4 | 74.0 |\n| Oracle | 82.5 | 80.3 | 69.2 | 92.6 | 87.4 | 77.3 | 90.5 | \u003Cbr>90.9 | 83.8 |\n| ICL | 75.8 | 70.7 | 59.3 | 86.1 | 80.2 | 67.8 | 83.5 | 84.7 | 76.0 |\n| C3PO | 77.9 | 74.1 | 63.4 | 88.1 | 81.7 | 71.9 | 85.4 | 86.0 | 78.6 |\n|  |  |  |  |  |  |  |  |  |  |\n| Router Tuning | 75.3 | 70.3 | 60.1 | 85.7 | 79.8 | 68.5 | 82.8 | 84.2 | 75.8 |\n| Oracle Tuning | 77.2 | 73.5 | 62.8 | 87.6 | 81.3 | 71.2 | 84.9 | 85.5 | 78.0 |\n| Prefix Tuning | 74.5 | 68.9 | 57.9 | 84.8 | 79.1 | 66.3 | 82.1 | 83.8 | 74.7 |\n| Prompt Tuning | 75.0 | 69.6 | 58.6 | 85.2 | 79.6 | 67.0 | 82.7 | 84.0 | 75.2 |\n| Dense BP | 76.1 | 71.4 | 59.8 | 86.5 | 80.5 | 69.2 | 83.8 | 84.9 | 76.5 |\n| RoMA (Ours) | 78.8 | 74.8 | 65.5 | 88.6 | 83.1 | 73.8 | 85.1 | 86.3 | 79.5 |\n\n\n| MMLU |  | Hella\u003Cbr>Swag | ARC-C ARC-E PIQA |  |  | Wino\u003Cbr>Grande | BoolQ GSM8K |  |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| ∼1B active parameters |  |  |  |  |  |  |  |  |\n| Llama3.2-1B | 27.4 | 57.9 | 32.1 | 53.9 | 72.4 | 57.4 | 63.7 | 39.4 |\n| OLMo-1B | 24.1 | 61.8 | 29.6 | 55.7 | 75.6 | 56.8 | 64.2 | 28.5 |\n| OLMoE-7B-A1B | 57.8 | 77.9 | 51.3 | 79.8 | 80.7 | 72.2 | 75.4 | 45.5 |\n| ∼3B active parameters |  |  |  |  |  |  |  |  |\n| Gemma2-3B | 43.7 | 66.3 | 58.4 | 75.2 | 71.8 | 64.5 | 73.1 | 41.4 |\n| DeepSeekMoE-16B-A3B | 46.2 | 78.0 | 50.3 | 73.8 | 79.9 | 70.1 | 72.3 | 62.2 |\n| Qwen3-30B-A3B | 74.2 | 68.5 | 56.8 | 84.3 | 78.5 | 65.2 | 81.3 | 83.4 |\n| ∼7-8B active parameters |  |  |  |  |  |  |  |  |\n| Qwen2-7B | 53.4 | 74.9 | 45.8 | 69.7 | 77.2 | 68.1 | 84.8 | 79.9 |\n| Mistral-7B | 59.6 | 81.0 | 53.8 | 79.6 | 82.2 | 74.0 | 68.1 | 37.9 |\n| Llama3.1-8B | 57.7 | 77.9 | 48.7 | 80.8 | 81.4 | 73.5 | 81.9 | 49.6 |\n| ∼13-14B active parameters |  |  |  |  |  |  |  |  |\n| Llama2-13B | 53.8 | 78.6 | 50.1 | 74.5 | 79.1 | 70.1 | 75.7 | 35.2 |\n| Vicuna-13B | 51.3 | 76.2 | 47.4 | 72.8 | 78.0 | 68.2 | 71.5 | 32.2 |\n| Qwen1.5-14B | 66.7 | 81.5 | 58.0 | 85.3 | 82.1 | 76.9 | 81.3 | 58.4 |\n| ∼27-34B active parameters |  |  ","cbCaikpNcxvNAc1o","https://ap.wps.com/l/cbCaikpNcxvNAc1o","pdf",1911722,1,20,"English","en",105,"# Experimental Results\n## Benchmark Scores Across Methods\n## Active-Parameter Regimes and Model Sizes","[{\"question\":\"What evaluation benchmarks are reported in the table?\",\"answer\":\"The table includes MMLU, HellaSwag, ARC-C, ARC-E, PIQA, WinoGrande, BoolQ, GSM8K, and an average column summarizing overall performance.\"},{\"question\":\"Which tuning strategies are compared against baseline models?\",\"answer\":\"The document compares ICL, C3PO, Router Tuning, Oracle Tuning, Prefix Tuning, Prompt Tuning, and Dense BP, alongside base models and RoMA (Ours).\"},{\"question\":\"How does RoMA (Ours) perform relative to other methods?\",\"answer\":\"RoMA (Ours) achieves higher scores than the base model and several tuning baselines across multiple benchmarks, and it shows improved average performance in the reported comparisons.\"}]","ICLR-2026-routing-manifold-alignment-improves-generalization-of-mixture-of-experts-llms-Paper-Conference - research results | PDF",1788358855,50,{"code":4,"msg":31,"data":32},"ok",{"site_id":24,"language":23,"slug":33,"title":13,"keywords":34,"description":14,"schema_data":35,"social_meta":87,"head_meta":89,"extra_data":91,"updated_unix":28},"iclr-2026-routing-manifold-alignment-improves-generalization-of-mixture-of-experts-llms-paper-conference-research-results","",{"@graph":36,"@context":86},[37,54,69],{"@type":38,"itemListElement":39},"BreadcrumbList",[40,44,48,51],{"item":41,"name":42,"@type":43,"position":20},"https://docshare.wps.com","Home","ListItem",{"item":45,"name":46,"@type":43,"position":47},"https://docshare.wps.com/document/","Document",2,{"item":49,"name":12,"@type":43,"position":50},"https://docshare.wps.com/document/research-report/",3,{"item":52,"name":13,"@type":43,"position":53},"https://docshare.wps.com/document/iclr-2026-routing-manifold-alignment-improves-generalization-of-mixture-of-experts-llms-paper-conference-research-results/184331/",4,{"url":52,"name":13,"@type":55,"author":56,"headline":13,"publisher":58,"fileFormat":61,"inLanguage":23,"description":14,"dateModified":62,"datePublished":63,"encodingFormat":61,"isAccessibleForFree":64,"interactionStatistic":65},"DigitalDocument",{"name":9,"@type":57},"Person",{"url":41,"name":59,"@type":60},"DocShare","Organization","application/pdf","2026-09-04","2026-09-02",true,{"@type":66,"interactionType":67,"userInteractionCount":20},"InteractionCounter",{"@type":68},"ViewAction",{"@type":70,"mainEntity":71},"FAQPage",[72,78,82],{"name":73,"@type":74,"acceptedAnswer":75},"What evaluation benchmarks are reported in the table?","Question",{"text":76,"@type":77},"The table includes MMLU, HellaSwag, ARC-C, ARC-E, PIQA, WinoGrande, BoolQ, GSM8K, and an average column summarizing overall performance.","Answer",{"name":79,"@type":74,"acceptedAnswer":80},"Which tuning strategies are compared against baseline models?",{"text":81,"@type":77},"The document compares ICL, C3PO, Router Tuning, Oracle Tuning, Prefix Tuning, Prompt Tuning, and Dense BP, alongside base models and RoMA (Ours).",{"name":83,"@type":74,"acceptedAnswer":84},"How does RoMA (Ours) perform relative to other methods?",{"text":85,"@type":77},"RoMA (Ours) achieves higher scores than the base model and several tuning baselines across multiple benchmarks, and it shows improved average performance in the reported comparisons.","https://schema.org",{"og:url":52,"og:type":88,"og:title":13,"og:site_name":59,"og:description":14},"article",{"robots":90,"canonical":52},"index,follow",{"doc_id":7,"site_id":24},{"code":4,"msg":5,"data":93},[94,98,102,106,111,115,120,123,127,130,134],{"id":20,"doc_module":4,"doc_module_name":46,"category_name":95,"show_sort_weight":96,"slug":97},"Story & Novel",90,"story-novel",{"id":47,"doc_module":4,"doc_module_name":46,"category_name":99,"show_sort_weight":100,"slug":101},"Literature",80,"literature",{"id":53,"doc_module":4,"doc_module_name":46,"category_name":103,"show_sort_weight":104,"slug":105},"Exam",70,"exam",{"id":107,"doc_module":4,"doc_module_name":46,"category_name":108,"show_sort_weight":109,"slug":110},5,"Comic",60,"comic",{"id":112,"doc_module":4,"doc_module_name":46,"category_name":113,"show_sort_weight":29,"slug":114},6,"Technology","technology",{"id":116,"doc_module":4,"doc_module_name":46,"category_name":117,"show_sort_weight":118,"slug":119},7,"Healthcare",40,"healthcare",{"id":11,"doc_module":4,"doc_module_name":46,"category_name":12,"show_sort_weight":121,"slug":122},30,"research-report",{"id":124,"doc_module":4,"doc_module_name":46,"category_name":125,"show_sort_weight":21,"slug":126},9,"Religion & Spirituality","religion-spirituality",{"id":21,"doc_module":4,"doc_module_name":46,"category_name":128,"show_sort_weight":21,"slug":129},"World Cup","world-cup",{"id":131,"doc_module":4,"doc_module_name":46,"category_name":132,"show_sort_weight":131,"slug":133},10,"Lifestyle","lifestyle",{"id":135,"doc_module":4,"doc_module_name":46,"category_name":136,"show_sort_weight":107,"slug":137},19,"General","general"]