[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-290373-105":3,"doc-detail-290373-en":85,"detail-sidebar-cat-0-en-105":102},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":78,"head_meta":80,"extra_data":82,"updated_unix":84},105,"en","accelerating-collective-communication-for-distributed-machine-learning-dissertation","Accelerating Collective Communication for Distributed Machine Learning - Dissertation","","Collective communication has become a core enabler for distributed machine learning, but it also limits latency, throughput, and ultimately hardware scalability when training or serving large language models. This dissertation optimizes collective communication across network topology design, communication scheduling, and parallelization strategies. It presents topology-scheduling co-design for direct-connect optical circuit networks, including expansion techniques and linear-programming schedule synthesis. It also introduces ForestColl, a polynomial-time schedule generator that achieves throughput optimality for any topology with large speedups.",{"@graph":14,"@context":77},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/accelerating-collective-communication-for-distributed-machine-learning-dissertation/290373/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/accelerating-collective-communication-for-distributed-machine-learning-dissertation/290373.png","ImageObject",300,407,{"name":42,"@type":43},"Eliana","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-07","2026-09-17",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",18,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69,73],{"name":60,"@type":61,"acceptedAnswer":62},"What problem does the dissertation address in distributed machine learning?","Question",{"text":63,"@type":64},"It addresses collective communication as a performance bottleneck that limits efficient utilization and scalability of hardware when training or serving large language models.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How does the dissertation optimize collective communication for direct-connect optical circuit networks?",{"text":68,"@type":64},"It co-designs network topology and communication scheduling using expansion techniques and a linear-programming-based schedule generation algorithm to synthesize efficient large-scale topologies and schedules.",{"name":70,"@type":61,"acceptedAnswer":71},"What is ForestColl and why is it important?",{"text":72,"@type":64},"ForestColl is a schedule generation algorithm that produces throughput-optimal schedules for any network topology in polynomial time by constructing spanning trees for collective communications, achieving major speedups over prior approaches.",{"name":74,"@type":61,"acceptedAnswer":75},"How does the dissertation propose future work for automatic parallelization?",{"text":76,"@type":64},"It proposes searching for parallelization and optimization strategies by using tensor sharding and processing states inside the compiled computation graph, aiming to discover optimal strategies through abstract tensor state determination.","https://schema.org",{"og:url":32,"og:type":79,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":81,"canonical":32},"index,follow",{"doc_id":83,"site_id":7},290373,1789641520,{"code":4,"msg":86,"data":87},"success",{"doc_id":83,"user_id":88,"nickname":42,"user_avatar":89,"doc_module":4,"category_id":90,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":91,"file_id":92,"file_url":93,"file_type":94,"file_size":95,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":96,"language":97,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":98,"faqs":99,"seo_title":100,"seo_description":12,"update_tm":84,"read_time":101,"preview_page_count":4},4398048949847,"https://ap-avatar.wpscdn.com/avatar/400002536579ef2da7f?_k=1778318612642679267",8,"Accelerating Collective Communication for Distributed Machine Learning  \nLiangyu Zhao  \nA dissertation submitted in partial fulfillment of the  \nrequirements for the degree of  \nDoctor of Philosophy  \nUniversity of Washington  \n2026  \nReading Committee:  \nArvind Krishnamurthy, Chair  \nRatul Mahajan  \nBaris Kasikci  \nProgram Authorized to Offer Degree:  \nPaul G. Allen School of Computer Science & Engineering  \n©Copyright 2026 Liangyu Zhao  \nUniversity of Washington  \nAbstract  \nAccelerating Collective Communication for Distributed Machine Learning  \nLiangyu Zhao  \nChair of the Supervisory Committee:  \nArvind Krishnamurthy  \nPaul G. Allen School of Computer Science & Engineering  \nCollective communication has emerged as a cornerstone of distributed machine learning, enabling datacenterscale clusters of accelerators to collaboratively train or serve large language models. However, it has also become a significant performance bottleneck, impeding the efficient utilization and scalability of hardware resources. This dissertation focuses on optimizing collective communication for machine learning hardware and workloads, approaching the challenge from the perspectives of network topology, communication scheduling, and parallelization strategies.  \nWe first present our work on co-optimizing network topology and communication scheduling for directconnect optical circuit networks. We propose expansion techniques and a linear programming-based schedule generation algorithm to synthesize efficient large-scale topologies and schedules, thereby forming a Pareto frontier of the latency-throughput trade-off. Our approach enables efficient collective communication on low-diameter topologies.  \nThen, we introduce ForestColl, a schedule generation algorithm capable of producing throughput-optimal schedules for any network topology in polynomial time. ForestColl leverages prior graph-theoretical results to construct spanning trees for collective communications. It is the first work to achieve throughput optimality for collective communications while delivering orders-of-magnitude speedups in schedule generation compared to prior approaches.  \nFinally, we outline our future work on automating the search for parallelization and optimization strategies in machine learning training. We propose a strategy grounded in the sharding and processing states of tensors within the compiled computation graph. By adopting a unified view of all tensor types, we propose a method that can discover optimal parallelization and optimization strategies through the determination of abstract tensor states.  \nTable of Contents  \nPage  \nList of Figures . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . iii  \nList of Tables . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . v  \nChapter 1: Introduction ....................................... 1  \n1.1 Collective Communication in Distributed Machine Learning ................. 1  \n1.2 Thesis Statement & Contributions ............................... 2  \n1.3 Published Works ........................................ 3  \nChapter 2: Background ....................................... 4  \n2.1 Collective Operations ...................................... 4  \n2.2 Network Topologies in Distributed Machine Learning Systems ................ 5  \nChapter 3: Efficient Direct-Connect Topologies for Collective Communications ......... 7  \n3.1 Introduction ........................................... 7  \n3.2 Background & Related Work .................................. 9  \n3.3 Formal Model of Collective Communications ......................... 11  \n3.4 Overview of Our Approach ................................... 14  \n3.5 Expansion Techniques ..................................... 15  \n3.6 Breadth-First-Broadcast (BFB) Schedule ............................ 22  \n3.7 Schedule Compilation ...................................... 25  \n3.8 Evaluation .......................","cbCaih2TEB8hxtet","https://ap.wps.com/l/cbCaih2TEB8hxtet","pdf",11826314,158,"English","# Chapter 1: Introduction\n## 1.1 Collective Communication in Distributed Machine Learning\n## 1.2 Thesis Statement & Contributions\n## 1.3 Published Works\n# Chapter 2: Background\n## 2.1 Collective Operations\n## 2.2 Network Topologies in Distributed Machine Learning Systems\n# Chapter 3: Efficient Direct-Connect Topologies for Collective Communications\n## 3.1 Introduction\n## 3.2 Background & Related Work\n## 3.3 Formal Model of Collective Communications\n## 3.4 Overview of Our Approach\n## 3.5 Expansion Techniques\n## 3.6 Breadth-First-Broadcast (BFB) Schedule\n## 3.7 Schedule Compilation\n## 3.8 Evaluation\n## 3.9 Concluding Remarks\n# Chapter 4: ForestColl: Throughput-Optimal Collective Communications on Heterogeneous Network Fabrics\n## 4.1 Introduction\n## 4.2 Background & Related Work\n## 4.3 Overview of ForestColl\n## 4.4 Throughput Optimality for Collectives\n## 4.5 Algorithm Design\n## 4.6 Evaluation\n## 4.7 Concluding Remarks\n# Chapter 5: Future Work: Automatic Machine Learning Parallelization via Tensor State Search\n## 5.1 Introduction\n## 5.2 Tensor State\n## 5.3 Compute Graph\n## 5.4 Future Work: Tensor State Search\n# Chapter 6: Conclusion","[{\"question\":\"What problem does the dissertation address in distributed machine learning?\",\"answer\":\"It addresses collective communication as a performance bottleneck that limits efficient utilization and scalability of hardware when training or serving large language models.\"},{\"question\":\"How does the dissertation optimize collective communication for direct-connect optical circuit networks?\",\"answer\":\"It co-designs network topology and communication scheduling using expansion techniques and a linear-programming-based schedule generation algorithm to synthesize efficient large-scale topologies and schedules.\"},{\"question\":\"What is ForestColl and why is it important?\",\"answer\":\"ForestColl is a schedule generation algorithm that produces throughput-optimal schedules for any network topology in polynomial time by constructing spanning trees for collective communications, achieving major speedups over prior approaches.\"},{\"question\":\"How does the dissertation propose future work for automatic parallelization?\",\"answer\":\"It proposes searching for parallelization and optimization strategies by using tensor sharding and processing states inside the compiled computation graph, aiming to discover optimal strategies through abstract tensor state determination.\"}]","Accelerating Collective Communication for Distributed Machine Learning - Dissertation | PDF",398,{"code":4,"msg":86,"data":103},[104,108,112,116,121,126,131,134,139,142,146],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":105,"show_sort_weight":106,"slug":107},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":109,"show_sort_weight":110,"slug":111},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":113,"show_sort_weight":114,"slug":115},"Exam",70,"exam",{"id":117,"doc_module":4,"doc_module_name":25,"category_name":118,"show_sort_weight":119,"slug":120},5,"Comic",60,"comic",{"id":122,"doc_module":4,"doc_module_name":25,"category_name":123,"show_sort_weight":124,"slug":125},6,"Technology",50,"technology",{"id":127,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":129,"slug":130},7,"Healthcare",40,"healthcare",{"id":90,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":132,"slug":133},30,"research-report",{"id":135,"doc_module":4,"doc_module_name":25,"category_name":136,"show_sort_weight":137,"slug":138},9,"Religion & Spirituality",20,"religion-spirituality",{"id":137,"doc_module":4,"doc_module_name":25,"category_name":140,"show_sort_weight":137,"slug":141},"World Cup","world-cup",{"id":143,"doc_module":4,"doc_module_name":25,"category_name":144,"show_sort_weight":143,"slug":145},10,"Lifestyle","lifestyle",{"id":147,"doc_module":4,"doc_module_name":25,"category_name":148,"show_sort_weight":117,"slug":149},19,"General","general"]