[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-644980-105":59,"doc-detail-644980-en":129},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":122,"head_meta":124,"extra_data":126,"updated_unix":128},105,"en","ikea-manuals-at-work-4d-grounding-of-assembly-instructions-on-internet-videos","IKEA Manuals at Work - 4D Grounding of Assembly Instructions on Internet Videos","","Shape assembly is a ubiquitous daily-life task essential for building complex 3D structures such as IKEA furniture. While autonomous agents for shape assembly have progressed, existing datasets do not address 4D grounding of assembly instructions in videos. This work introduces IKEA Video Manuals, providing 3D furniture-part models, internet assembly videos, and dense spatio-temporal alignments across modalities. It supports assembly plan generation, part-conditioned segmentation, pose estimation, video object segmentation, and end-to-end furniture assembly with evaluation metrics and baselines.",{"@graph":69,"@context":121},[70,84,104],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/ikea-manuals-at-work-4d-grounding-of-assembly-instructions-on-internet-videos/644980/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":98,"encodingFormat":97,"isAccessibleForFree":99,"interactionStatistic":100},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/ikea-manuals-at-work-4d-grounding-of-assembly-instructions-on-internet-videos/644980.png","ImageObject",300,407,{"name":92,"@type":93},"Putri","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-23",true,{"@type":101,"interactionType":102,"userInteractionCount":4},"InteractionCounter",{"@type":103},"ViewAction",{"@type":105,"mainEntity":106},"FAQPage",[107,113,117],{"name":108,"@type":109,"acceptedAnswer":110},"What gap does the paper address in existing assembly datasets?","Question",{"text":111,"@type":112},"Existing datasets do not tackle 4D grounding of assembly instructions in videos, which is needed for understanding how assembly evolves over time in 3D space.","Answer",{"name":114,"@type":109,"acceptedAnswer":115},"What is included in the IKEA Video Manuals dataset?",{"text":116,"@type":112},"It provides 3D models of furniture parts, instructional assembly videos from the internet, and annotations for dense spatio-temporal alignments, including 2D-3D correspondences, temporal step alignments, and part segmentation.",{"name":118,"@type":109,"acceptedAnswer":119},"Which applications and evaluation components does the paper present using the dataset?",{"text":120,"@type":112},"The paper presents applications such as assembly plan generation, part-conditioned segmentation, part-conditioned pose estimation, video object segmentation, and furniture assembly based on instructional video manuals, along with evaluation metrics and baseline methods.","https://schema.org",{"og:url":83,"og:type":123,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":125,"canonical":83},"index,follow",{"doc_id":127,"site_id":62},644980,1792727133,{"code":4,"msg":5,"data":130},{"doc_id":127,"user_id":131,"nickname":92,"user_avatar":132,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":133,"file_id":134,"file_url":135,"file_type":136,"file_size":137,"view_count":4,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":138,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":128,"read_time":143},962085571259,"https://ap-avatar.wpscdn.com/davatar_29158cc5080c5b710cf443261637dec0","IKEA Manuals at Work: 4D Grounding of Assembly Instructions on Internet Videos  \nYunong Liu 1 Juan Carlos Niebles 1  \nCristobal Eyzaguirre 1 Manling Li 1 Vineeth Ravi2 Saumitra Mishra2  \nShubh Khanna 1  \nWeiyu Liu 1 ∗ Jiajun Wu 1 ∗  \n1 Stanford University 2J.P. Morgan AI Research  \nAbstract  \nShape assembly is a ubiquitous task in daily life, integral for constructing complex 3D structures like IKEA furniture. While significant progress has been made in developing autonomous agents for shape assembly, existing datasets have not yet tackled the 4D grounding of assembly instructions in videos, essential for a holistic understanding of assembly in 3D space over time. We introduce IKEA Video Manuals, a dataset that features 3D models of furniture parts, instructional manuals, assembly videos from the Internet, and most importantly, annotations of dense spatio-temporal alignments between these data modalities. To demonstrate the utility of IKEA Video Manuals, we present five applications essential for shape assembly: assembly plan generation, part-conditioned segmentation, partconditioned pose estimation, video object segmentation, and furniture assembly based on instructional video manuals. For each application, we provide evaluation metrics and baseline methods. Through experiments on our annotated data, we highlight many challenges in grounding assembly instructions in videos to improve shape assembly, including handling occlusions, varying viewpoints, and extended assembly sequences.  \n1 Introduction  \nThe autonomous assembly of complex 3D structures requires an understanding at multiple levels of abstraction. The top level is task planning—decomposing the task into subtasks and computing their dependency and ordering, as outlined in an instruction manual; the middle level is visual grounding—registering each part with perceptual input at the pixel level, identifying their geometry and pose, so that the manual instructions get translated to actionable steps; the bottom level is motion planning and control—executing the steps based on the instructions and the identified states, avoiding collisions given the specific embodiment. A successful assembly requires solving all of these problems together, which is difficult. Thus, assembly remains a significant challenge in AI and robotics.  \nWhen developing assembly benchmarks, researchers often choose IKEA furniture as target objects due to its ubiquity and standardization. However, existing benchmarks and datasets focus only on part of the assembly problem. Classic work on designing assembly instructions provides step-by-step guidance for task planning, but they are not visually grounded [1]; recent work has attempted to ground assembly steps to 3D part models and register them at the pixel level [2], but it does not include action trajectories of how the assembly may actually happen; datasets that provide 3D groundingsin the form RGB-D videos are collected in controlled lab environments [3, 4]; a video dataset of IKEA furniture assembly from the Internet includes more diverse demonstrations [5], but it lacks correspondence with 3D part models and alignment with the instruction manuals.  \nTo address these limitations, we introduce the IKEA Video Manuals dataset, a large-scale multimodal dataset with high-quality, spatial-temporal alignments of step-by-step instructions, 3D models, and  \n∗ Equal advising. Project page: [yunongliu1.github.io/ikea-video-manual](yunongliu1.github.io/ikea-video-manual)  \n38th Conference on Neural Information Processing Systems (NeurIPS 2024) Track on Datasets and Benchmarks.  \n(a) Manual Images  \n(b) Frames  \n(c) Masks  \n(d) 3D Poses  \n(e) Parts in Frame  \nStep2 Step3 Step4  \n. . .  \n. . .  \n. . . Substepi Substepi+1 Substepi+2 . . .  \nFigure 1: Dataset Overview. (a) Manual images showing the assembly steps. (b) Video frames from the corresponding assembly videos. Temporal alignment between the video frames and each assembly step is also provided. (c) Seg","cbCaibdhk5ffp2hE","https://ap.wps.com/l/cbCaibdhk5ffp2hE","pdf",35780009,31,"English","# Introduction\n## Related Work\n### Instructional Video Datasets and Procedural Understanding","[{\"question\":\"What gap does the paper address in existing assembly datasets?\",\"answer\":\"Existing datasets do not tackle 4D grounding of assembly instructions in videos, which is needed for understanding how assembly evolves over time in 3D space.\"},{\"question\":\"What is included in the IKEA Video Manuals dataset?\",\"answer\":\"It provides 3D models of furniture parts, instructional assembly videos from the internet, and annotations for dense spatio-temporal alignments, including 2D-3D correspondences, temporal step alignments, and part segmentation.\"},{\"question\":\"Which applications and evaluation components does the paper present using the dataset?\",\"answer\":\"The paper presents applications such as assembly plan generation, part-conditioned segmentation, part-conditioned pose estimation, video object segmentation, and furniture assembly based on instructional video manuals, along with evaluation metrics and baseline methods.\"}]","IKEA Manuals at Work - 4D Grounding of Assembly Instructions on Internet Videos | PDF",78]