[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-648480-105":59,"doc-detail-648480-en":134},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":127,"head_meta":129,"extra_data":131,"updated_unix":133},105,"en","communicating-design-intent-using-drawing-and-text-paper","Communicating Design Intent Using Drawing and Text - paper","","Realizing a designer’s intent in software currently requires tedious manipulation of geometric primitives, such as points and curves. In contrast, designers communicate abstract goals through natural language combined with drawings. To enable human–machine collaboration in design, the study establishes initial benchmarks using an online designer–maker task. Pairs of humans exchanged iterative messages with language, drawings, or both to recreate target designs, revealing early preference for drawings and multimodal strategies. Comparing human makers to GPT-4V identifies a gap in understanding multimodal instructions and proposes a bridging path.",{"@graph":69,"@context":126},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/communicating-design-intent-using-drawing-and-text-paper/648480/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/communicating-design-intent-using-drawing-and-text-paper/648480.png","ImageObject",300,407,{"name":92,"@type":93},"Rizky","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-24","2026-10-23",true,{"@type":102,"interactionType":103,"userInteractionCount":14},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118,122],{"name":109,"@type":110,"acceptedAnswer":111},"What problem does the paper address?","Question",{"text":112,"@type":113},"It addresses how to realize a designer’s intent in software without tedious low-level geometric editing by enabling systems that understand how designers communicate intent using drawings and text.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How was the study conducted?",{"text":117,"@type":113},"The paper reports an online study where designer–maker pairs collaborated over multiple turns to recreate target designs using messages containing language, drawings, or both.",{"name":119,"@type":110,"acceptedAnswer":120},"What did the authors find about multimodal communication?",{"text":121,"@type":113},"They found an early preference for communicating with drawings and observed several multimodal strategies for conveying design intent.",{"name":123,"@type":110,"acceptedAnswer":124},"How does GPT-4V compare to human makers?",{"text":125,"@type":113},"By comparing how human makers and GPT-4V carried out instructions, the paper identifies a gap in human and machine understanding of multimodal instructions and suggests a path to bridge it.","https://schema.org",{"og:url":83,"og:type":128,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":130,"canonical":83},"index,follow",{"doc_id":132,"site_id":62},648480,1792842736,{"code":4,"msg":5,"data":135},{"doc_id":132,"user_id":136,"nickname":92,"user_avatar":137,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":138,"file_id":139,"file_url":140,"file_type":141,"file_size":142,"view_count":14,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":39,"language":143,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":144,"faqs":145,"seo_title":146,"seo_description":67,"update_tm":147,"read_time":46},962085564807,"https://ap-avatar.wpscdn.com/davatar_6f874abed73319feea01a86fa6f0fab8","Communicating Design Intent Using Drawing and Text  \nWilliam P. McCarthy  \nUniversity of California, San Diego San Diego, USA Autodesk San Francisco, USA [william.mccarthy@autodesk.com](william.mccarthy@autodesk.com)  \nJustin Matejka  \nAutodesk Research Toronto, Canada [justin.matejka@autodesk.com](justin.matejka@autodesk.com)  \nKarl D. D. Willis  \nAutodesk San Francisco, USA [karl.willis@autodesk.com](karl.willis@autodesk.com)  \nJudith E. Fan  \nStanford University, Department of Psychology Stanford, USA [jefan@stanford.edu](jefan@stanford.edu)  \nYewen Pu  \nAutodesk San Francisco, USA [yewen.pu@autodesk.com](yewen.pu@autodesk.com)  \nABSTRACT  \nRealizing a designer’s intent in software currently requires tedious manipulation of geometric primitives, such as points and curves. By contrast, designers routinely communicate more abstract design goals to one another using an efficient combination of natural language and drawings. What would it take to develop artificial systems that understand how humans naturally convey design intent, and thereby enable more seamless interactions between humans and machines throughout the design process? First, it is vital to establish benchmarks that showcase the full range of strategies that humans use to successfully communicate about design intent. Here we take initial steps towards that goal by conducting an online study in which pairs of human participants – a “Designer” and “Maker” – collaborated over multiple turns to recreate target designs. In each turn, Designers sent messages containing language, drawings, or both to the Maker, describing how to modify an existing design toward the target. We found a preference for communicating using drawings in early turns and observed several multimodal strategies for conveying design intent. By comparing how human Makers and GPT-4V carried out instructions, we identify a gap in human and machine understanding of multimodal instructions and suggest a path for bridging this gap.  \nACM Reference Format:  \nWilliam P. McCarthy, Justin Matejka, Karl D. D. Willis, Judith E. Fan, andYewen Pu. 2024. Communicating Design Intent Using Drawing and Text. In Creativity and Cognition (C&C’24), June 23–26, 2024, Chicago, IL, USA. ACM, New York, NY, USA, 8 pages. [https://doi.org/10.1145/3635636.3664261](https://doi.org/10.1145/3635636.3664261)  \n1 INTRODUCTION  \nOur world is filled with designed objects, from physical artifacts such as bicycles and buildings, to visual media such as icons and diagrams. Designs are typically represented in machines as lowlevel geometries (e.g. lines and arcs) and are manipulated directly  \nPermission to make digital or hard copies of part or all ofthis work for personal or classroom use is granted without fee provided that copies are not made or distributed for profit or commercial advantage and that copies bear this notice and the full citation on the first page. Copyrights for third-party components of this work must be honored. For all other uses, contact the owner/author(s) .  \nC&C’24, June 23–26, 2024, Chicago, IL, USA © 2024 Copyright held by the owner/author(s) .  \nACM ISBN 979-8-4007-0485-7/24/06  \n[https://doi.org/10.1145/3635636.3664261](https://doi.org/10.1145/3635636.3664261)  \nusing a mouse and keyboard– a laborious process. This process differs dramatically from how designers communicate design intent to each other. Firstly, designers use a combination of multiple communication modalities, typically words and drawings [11, 17], to communicate intents efficiently. Secondly, the initial realization of a concept is typically just the beginning– designers go back and forth over several iterations, updating their design to explore a problem space, adapt to changing constraints, and to repair errors made during communication. Recently developed foundation models capable of understanding and generating text [4], images [3, 12], and computer programs [13] present new opportunities for interacting with digital representations o","cbCaiiUzqXEoJXoP","https://ap.wps.com/l/cbCaiiUzqXEoJXoP","pdf",2210828,"English","# Introduction\n# Abstract","[{\"question\":\"What problem does the paper address?\",\"answer\":\"It addresses how to realize a designer’s intent in software without tedious low-level geometric editing by enabling systems that understand how designers communicate intent using drawings and text.\"},{\"question\":\"How was the study conducted?\",\"answer\":\"The paper reports an online study where designer–maker pairs collaborated over multiple turns to recreate target designs using messages containing language, drawings, or both.\"},{\"question\":\"What did the authors find about multimodal communication?\",\"answer\":\"They found an early preference for communicating with drawings and observed several multimodal strategies for conveying design intent.\"},{\"question\":\"How does GPT-4V compare to human makers?\",\"answer\":\"By comparing how human makers and GPT-4V carried out instructions, the paper identifies a gap in human and machine understanding of multimodal instructions and suggests a path to bridge it.\"}]","Communicating Design Intent Using Drawing and Text - paper | PDF",1792793684]