[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-1-en-105":3,"doc-seo-238506-105":53,"doc-detail-238506-en":127},{"code":4,"msg":5,"data":6},0,"success",[7,14,19,24,29,34,39,44,49],{"id":8,"doc_module":9,"doc_module_name":10,"category_name":11,"show_sort_weight":12,"slug":13},11,1,"Template","Presentations",90,"presentations",{"id":15,"doc_module":9,"doc_module_name":10,"category_name":16,"show_sort_weight":17,"slug":18},12,"Resumes",80,"resumes",{"id":20,"doc_module":9,"doc_module_name":10,"category_name":21,"show_sort_weight":22,"slug":23},14,"Invoices",70,"invoices",{"id":25,"doc_module":9,"doc_module_name":10,"category_name":26,"show_sort_weight":27,"slug":28},15,"Posters",60,"posters",{"id":30,"doc_module":9,"doc_module_name":10,"category_name":31,"show_sort_weight":32,"slug":33},16,"Social Media",50,"social-media",{"id":35,"doc_module":9,"doc_module_name":10,"category_name":36,"show_sort_weight":37,"slug":38},17,"Forms",40,"forms",{"id":40,"doc_module":9,"doc_module_name":10,"category_name":41,"show_sort_weight":42,"slug":43},18,"Letters",30,"letters",{"id":45,"doc_module":9,"doc_module_name":10,"category_name":46,"show_sort_weight":47,"slug":48},21,"Paper Templates",5,"papers-templates",{"id":50,"doc_module":9,"doc_module_name":10,"category_name":51,"show_sort_weight":4,"slug":52},158,"General","general-158",{"code":4,"msg":54,"data":55},"ok",{"site_id":56,"language":57,"slug":58,"title":59,"keywords":60,"description":61,"schema_data":62,"social_meta":120,"head_meta":122,"extra_data":124,"updated_unix":126},105,"en","nest-network-and-memory-aware-device-placement-for-distributed-deep-learning-core-performance-results","NEST - Network-and Memory-Aware Device Placement for Distributed Deep Learning - Core performance results","","Distributed deep learning suffers from inefficient device placement and hybrid parallelism selection because many frameworks use simplified, topology-agnostic searches and address memory constraints only after placement. NEST introduces a network-, compute-, and memory-aware framework that incrementally models communication and memory in a structured dynamic program, jointly optimizing hybrid parallelism and device placement. The approach captures realistic datacenter hierarchies, enables provable optimality, detects bottlenecks early, and achieves up to 2.4× higher throughput while scaling to clusters with 1,000+ GPUs.",{"@graph":63,"@context":119},[64,80,102],{"@type":65,"itemListElement":66},"BreadcrumbList",[67,71,74,77],{"item":68,"name":69,"@type":70,"position":9},"https://docshare.wps.com","Home","ListItem",{"item":72,"name":10,"@type":70,"position":73},"https://docshare.wps.com/template/",2,{"item":75,"name":11,"@type":70,"position":76},"https://docshare.wps.com/template/presentations/",3,{"item":78,"name":59,"@type":70,"position":79},"https://docshare.wps.com/template/nest-network-and-memory-aware-device-placement-for-distributed-deep-learning-core-performance-results/238506/",4,{"url":78,"name":59,"@type":81,"image":82,"author":87,"headline":59,"publisher":90,"fileFormat":93,"inLanguage":57,"description":61,"dateModified":94,"datePublished":95,"encodingFormat":93,"isAccessibleForFree":96,"interactionStatistic":97},"DigitalDocument",{"url":83,"@type":84,"width":85,"height":86},"https://docshare.wps.com/thumbnails/nest-network-and-memory-aware-device-placement-for-distributed-deep-learning-core-performance-results/238506.png","ImageObject",442,249,{"name":88,"@type":89},"Patrick","Person",{"url":68,"name":91,"@type":92},"DocShare","Organization","application/pdf","2026-11-03","2026-09-11",true,{"@type":98,"interactionType":99,"userInteractionCount":101},"InteractionCounter",{"@type":100},"ViewAction",8,{"@type":103,"mainEntity":104},"FAQPage",[105,111,115],{"name":106,"@type":107,"acceptedAnswer":108},"What problem does NEST address in distributed deep learning?","Question",{"text":109,"@type":110},"Efficient distributed training is bottlenecked by placement frameworks that ignore network topology and handle memory constraints only after placement, causing excessive synchronization, over-sharding, and poor scalability.","Answer",{"name":112,"@type":107,"acceptedAnswer":113},"How does NEST optimize device placement and parallelism?",{"text":114,"@type":110},"NEST incrementally models communication and memory within a structured dynamic program to jointly optimize hybrid parallelism and device placement on realistic datacenter topologies.",{"name":116,"@type":107,"acceptedAnswer":117},"What performance gains does NEST report?",{"text":118,"@type":110},"NEST achieves up to 2.4× higher throughput than state-of-the-art baselines (e.g., Alpa) and sustains scalability on clusters with over 1,000 GPUs.","https://schema.org",{"og:url":78,"og:type":121,"og:title":59,"og:site_name":91,"og:description":61},"article",{"robots":123,"canonical":78},"index,follow",{"doc_id":125,"site_id":56},238506,1791740550,{"code":4,"msg":5,"data":128},{"doc_id":125,"user_id":129,"nickname":88,"user_avatar":130,"doc_module":9,"category_id":8,"category_name":11,"doc_title":59,"doc_description":61,"doc_content":131,"file_id":132,"file_url":133,"file_type":134,"file_size":135,"view_count":101,"is_deleted":4,"is_public":9,"is_downloadable":9,"audit_status":9,"page_count":9,"language":136,"language_code":57,"site_id":56,"html_lang":57,"table_of_contents":137,"faqs":138,"seo_title":139,"seo_description":61,"update_tm":140,"read_time":4},549758146520,"https://ap-avatar.wpscdn.com/avatar/80002397d8c0411e94?_k=1775819394049821470","NEST: Network-and Memory-Aware Device Placement for Distributed Deep Learning  \nIrene Wang, Vishnu Venkata, Arvind Krishnamurthy, Divya Mahajan  \nProblem: Efficient distributed training is bottlenecked by frameworks that rely on simplified, topology-agnostic search and handle memory constraints post-hoc, leading to excessive synchronization, over-sharding, and poor scalability on realistic, hierarchical datacenter networks.  \nOur Approach: We introduce NEST, a network- , compute- , and memory-aware framework that incrementally models communication and memory within a structured dynamic program to jointly optimize hybrid parallelism and device placement with provable optimality on realistic datacenter topologies.  \nFindings: NEST demonstrates that explicitly modeling network hierarchy and memory constraints enables superior placement strategies, achieving up to 2.4× higher throughput than state-of-the-art baselines and sustaining performance scalability on clusters with over 1,000 GPUs.  \nPaper Link  \nHow to Train DNN Efficiently?  \nToday’s AI models are trained at massive scales. Communication can become the bottleneck.  \nOptimal partitioning and placement requires being aware of  \nthe underlying network.  \nNetwork- & Memory-Awareness  \nPrior works fail to consider Network and Memory realities atthe sametime and lead to suboptimal placement plans  \nOversimplify Networks  Post-Placement Memory Check  \nThe NEST Framework  \nKey Insights  \nTopology-adaptive partitioning balances latencies across uneven network links  \nExplicit memory modeling enables early bottleneck detection and scalable training  \nTemplate-based parallelism matches finegrained sharding on repetitive Transformer architectures  \nValidated to exceed profiling-based techniques on real-world clusters  \nResults  \nNEST’s distribution strategy achieves 2.4x higher throughput than state-of-the-art baseline Alpa  \n Manual  MCMC  Alpa-E  Phaze  NEST  \nThroughput Improvement  \n24  \n18  \n12  \n6  \n0  \n\n| Llam |  |  | a3- | 70B |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n|  |  | X \u003Cbr>| | |  |  |  |  |  | |  |  |  |  |  |  |  |  |  |  |  |\n|  | X |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  | X |  |  |  |\n\n64 128 256 512 1024  \n12  \n10  \n8  \n6  \n4  \n2  \n0  \n\n| GPT | 3- | 5B |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n| X X \u003Cbr>| |  |  | |  |  |  |  |  |  |  |  |  |  |  |  |  | X |  |  |  |\n\n17  \n64 128 256 512 1024  \n16  \n12  \n8  \n4  \n0  \n\n| Mixt |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n|  |  |  | ral 7B | x 8 |  |  |  |  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |\n|  |  | | | |  |  | |  |  |  |  |  |  |  |  |  |  |\n|  | X |  |  |  |  |  |  |  |  |  |  |  |  | X |  |  |  |\n\n64 128 256 512 1024  \nNumber of Devices","cbCaibhZ2A9CV2T2","https://ap.wps.com/l/cbCaibhZ2A9CV2T2","pdf",635109,"English","# Problem\n## Bottlenecks in distributed training\n# NEST Framework\n## Network- , compute- , and memory-aware modeling\n## Incremental optimization via structured dynamic programming\n# Key Insights\n## Topology-adaptive partitioning\n## Explicit memory modeling for early bottleneck detection\n## Template-based parallelism for Transformer sharding\n# Results\n## Throughput improvement over baselines\n## Scalability on large GPU clusters","[{\"question\":\"What problem does NEST address in distributed deep learning?\",\"answer\":\"Efficient distributed training is bottlenecked by placement frameworks that ignore network topology and handle memory constraints only after placement, causing excessive synchronization, over-sharding, and poor scalability.\"},{\"question\":\"How does NEST optimize device placement and parallelism?\",\"answer\":\"NEST incrementally models communication and memory within a structured dynamic program to jointly optimize hybrid parallelism and device placement on realistic datacenter topologies.\"},{\"question\":\"What performance gains does NEST report?\",\"answer\":\"NEST achieves up to 2.4× higher throughput than state-of-the-art baselines (e.g., Alpa) and sustains scalability on clusters with over 1,000 GPUs.\"}]","NEST - Network-and Memory-Aware Device Placement for Distributed Deep Learning - Core performance results | PDF",1789136820]