[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-516154-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-516154-en":130},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","human-object-interaction-via-automatically-designed-vlm-guided-motion-policy","HUMAN-OBJECT INTERACTION VIA AUTOMATICALLY DESIGNED VLM-GUIDED MOTION POLICY","","Human-object interaction (HOI) synthesis is crucial for animation, simulation, and robotics, yet existing solutions are constrained by costly motion capture requirements or manual reward engineering, limiting scalability and generalization. This paper presents a unified physics-based HOI framework using vision-language models to support long-horizon interactions across static, dynamic, and articulated objects. It introduces VLM-Guided Relative Movement Dynamics (RMD) for fine-grained spatio-temporal reward and goal construction, and a new Interplay dataset and extensive experiments show improved natural, human-like motion in both simple and complex multi-task settings.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/human-object-interaction-via-automatically-designed-vlm-guided-motion-policy/516154/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/human-object-interaction-via-automatically-designed-vlm-guided-motion-policy/516154.png","ImageObject",300,407,{"name":42,"@type":43},"miaandmiso","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-15","2026-10-06",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",10,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What problem does the paper address in human-object interaction synthesis?","Question",{"text":63,"@type":64},"It addresses scalability and generalizability limits caused by reliance on motion capture data or manual reward engineering in existing HOI approaches.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How does the framework use vision-language models for HOI?",{"text":68,"@type":64},"It leverages VLMs to provide motion-level, interaction-aware guidance by automatically constructing goal states and reward functions for reinforcement learning across long horizons.",{"name":70,"@type":61,"acceptedAnswer":71},"What is VLM-Guided Relative Movement Dynamics (RMD)?",{"text":72,"@type":64},"RMD is a structured spatio-temporal bipartite representation that encodes fine-grained relationships between human and object parts to ground VLM reasoning in motion-level interaction patterns.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},516154,1792170666,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,111,115,120,123,126],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":108,"show_sort_weight":109,"slug":110},7,"Healthcare",40,"healthcare",{"id":112,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":113,"slug":114},8,30,"research-report",{"id":116,"doc_module":4,"doc_module_name":25,"category_name":117,"show_sort_weight":118,"slug":119},9,"Religion & Spirituality",20,"religion-spirituality",{"id":118,"doc_module":4,"doc_module_name":25,"category_name":121,"show_sort_weight":118,"slug":122},"World Cup","world-cup",{"id":55,"doc_module":4,"doc_module_name":25,"category_name":124,"show_sort_weight":55,"slug":125},"Lifestyle","lifestyle",{"id":127,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":97,"slug":129},19,"General","general",{"code":4,"msg":82,"data":131},{"doc_id":79,"user_id":132,"nickname":42,"user_avatar":133,"doc_module":4,"category_id":112,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":139,"language":140,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":12,"update_tm":144,"read_time":145,"preview_page_count":33,"preview_urls":146},7971478544854,"https://ap-avatar.wpscdn.com/davatar_994ba38a5ba835b3df7d355c54d3ed8d","HUMAN-OBJECT INTERACTION VIA AUTOMATICALLY DESIGNED VLM-GUIDED MOTION POLICY  \nZekai Deng 1 Ye Shi 1 ,2 Kaiyang Ji 1 Lan Xu 1 Shaoli Huang3 Jingya Wang 1 ∗  \n1 ShanghaiTech University 2InstAdapt 3AgiBot  \n{dengzk2023,shiye,jiky2024,xulan1,[wangjingya](wangjingya}@shanghaitech.edu.cn)[}](wangjingya}@shanghaitech.edu.cn)[@shanghaitech.edu.cn](wangjingya}@shanghaitech.edu.cn) , [shaolihuang@agibot.com](shaolihuang@agibot.com)  \nABSTRACT  \nHuman-object interaction (HOI) synthesis is crucial for applications in animation, simulation, and robotics. However, existing approaches either rely on expensive motion capture data or require manual reward engineering, limiting their scalability and generalizability. In this work, we introduce the first unified physics-based HOI framework that leverages Vision-Language Models (VLMs) to enable longhorizon interactions with diverse object types—including static, dynamic, and articulated objects. We introduce VLM-Guided Relative Movement Dynamics (RMD), a fine-grained spatio-temporal bipartite representation that automatically constructs goal states and reward functions for reinforcement learning. By encoding structured relationships between human and object parts, RMD enables VLMs to generate semantically grounded, interaction-aware motion guidance without manual reward tuning. To support our methodology, we present Interplay, a novel dataset with thousands of long-horizon static and dynamic interaction plans. Extensive experiments demonstrate that our framework outperforms existing methods in synthesizing natural, human-like motions across both simple single-task and complex multi-task scenarios. For more details, please refer to our project webpage: [https://vlm-rmd.github.io/](https://vlm-rmd.github.io/) .  \n1 INTRODUCTION  \nHuman-Object Interaction (HOI) understanding is fundamental to advancing embodied AI systems in simulation, animation, and robotics. Existing approaches to HOI learning generally fall into two categories. The first paradigm employs motion tracking policies to imitate reference demonstrations (Wang et al., 2023; 2024a; Xu et al., 2025) . While these methods can reproduce specific interaction patterns, they face critical limitations: Their heavy reliance on high-quality motion capture data creates scalability bottlenecks, and their strict adherence to reference trajectories inherently restricts their ability to generate novel interactions beyond the training distribution.  \nThe second paradigm adopts a task-centric perspective, developing specialized policies for individual interactions like sitting (Chao et al., 2021; Zhang et al., 2022) or carrying (Xu et al., 2024b; Szot et al., 2021; Deitke et al., 2022; Gao et al., 2024) . However, these methods encounter two fundamental challenges. First, they require labor-intensive reward engineering by domain experts—a particularly demanding requirement given the complex dynamics and contact-rich nature of realistic HOI scenarios (Sutton & Barto, 2018) . This manual reward design process inherently limits generalizability across interaction types, creating a need for automated objective formulation. Second, constrained by artificially designed, single-objective reward mechanisms, trained policies often overfit to specific behavioral patterns. Producing task-compliant but biomechanically unrealistic motions that violate natural human kinematics.  \nRecent efforts such as Eureka (Ma et al., 2023) and Grove (Cui et al., 2025) leverage large language models (LLMs) to automatically generate reward function code. However, they rely on iterative search paradigms that are often sample-inefficient and computationally costly. Xiao et al. (2024) advances this line of work by unifying reward design through a “chain-of-contacts” abstraction, which  \n∗ Corresponding author.  \nPush the sofa in front of the TV.  \nPlace the laundry basket in front of the washing machine.  \nFigure 1: Our framework automatically constructs both goal states and reward ","cbCainFOaLzZ9SpZ","https://ap.wps.com/l/cbCainFOaLzZ9SpZ","pdf",5440000,28,"English","# Abstract\n# Introduction\n## HOI learning paradigms\n## Limitations of motion tracking and task-centric policies\n## Related work using LLMs for reward design\n## Proposed physics-based VLM-guided framework and Relative Movement Dynamics (RMD)","[{\"question\":\"What problem does the paper address in human-object interaction synthesis?\",\"answer\":\"It addresses scalability and generalizability limits caused by reliance on motion capture data or manual reward engineering in existing HOI approaches.\"},{\"question\":\"How does the framework use vision-language models for HOI?\",\"answer\":\"It leverages VLMs to provide motion-level, interaction-aware guidance by automatically constructing goal states and reward functions for reinforcement learning across long horizons.\"},{\"question\":\"What is VLM-Guided Relative Movement Dynamics (RMD)?\",\"answer\":\"RMD is a structured spatio-temporal bipartite representation that encodes fine-grained relationships between human and object parts to ground VLM reasoning in motion-level interaction patterns.\"}]","HUMAN-OBJECT INTERACTION VIA AUTOMATICALLY DESIGNED VLM-GUIDED MOTION POLICY | PDF",1791326627,71,[147,151,155,159],{"page_no":22,"url":148,"width":149,"height":150},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/1f7b8408a62ad77bca64b4da752d1546.webp",983,1272,{"page_no":26,"url":152,"width":153,"height":154},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/219dfe3898881cdfb25dc01d1f05217d.webp",982,1270,{"page_no":30,"url":156,"width":157,"height":158},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/d97986eb76cfe5e7e157a89ad93e36f0.webp",981,1269,{"page_no":33,"url":160,"width":161,"height":162},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/a70a6fe46aaf91344210ae1e8bd7a2bb.webp",980,1268]