[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-646898-105":59,"doc-detail-646898-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","mpchat-towards-multimodal-persona-grounded-conversation-abstract-and-introduction","MPCHAT - Towards Multimodal Persona-Grounded Conversation - Abstract and Introduction","","Build self-consistent personalized dialogue agents by extending persona from text-only facts or personality traits to multimodal episodic memories. Introduce MPCHAT, the first multimodal persona-based dialogue dataset that pairs persona elements with images and sentences. Propose three multimodal persona-grounded dialogue benchmarks: next response prediction, grounding persona prediction, and speaker identification. Experiments show statistically significant improvements on all tasks, highlighting multimodal persona’s value for dialogue comprehension and offering a high-quality research resource.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/mpchat-towards-multimodal-persona-grounded-conversation-abstract-and-introduction/646898/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/mpchat-towards-multimodal-persona-grounded-conversation-abstract-and-introduction/646898.png","ImageObject",300,407,{"name":92,"@type":93},"dawugda","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-24","2026-10-23",true,{"@type":102,"interactionType":103,"userInteractionCount":8},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What is MPCHAT and what does it contribute to persona-based dialogue research?","Question",{"text":112,"@type":113},"MPCHAT is a multimodal persona-based dialogue dataset that extends persona with both text and images to represent episodic memories. It provides grounded multimodal dialogue examples for research on multimodal persona in conversation.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does the paper define multimodal persona in this work?",{"text":117,"@type":113},"Multimodal persona is formulated as a set of image-sentence pairs that describe memorable moments. Each user response in the dialogue is grounded on a corresponding persona element.",{"name":119,"@type":110,"acceptedAnswer":120},"What benchmark tasks are proposed for multimodal persona-grounded dialogue understanding?",{"text":121,"@type":113},"The paper proposes three retrieval-based dialogue tasks: next response prediction, grounding persona prediction, and speaker identification. Incorporating multimodal persona yields statistically significant performance gains across all tasks.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},646898,1792876768,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":8,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":26},3985747870947,"https://ap-avatar.wpscdn.com/davatar_3d24733baf745e90a7e4bdd5f77d97b2","MPCHAT: Towards Multimodal Persona-Grounded Conversation  \nJaewoo Ahn1 Yeda Song1 Sangdoo Yun2,1 Gunhee Kim1  \n1 Seoul National University 2NAVER AI Lab  \n{jaewoo.ahn,[yeda.song}@vision.snu.ac.kr](yeda.song}@vision.snu.ac.kr) , [sangdoo.yun@navercorp.com](sangdoo.yun@navercorp.com) , [gunhee@snu.ac.kr](gunhee@snu.ac.kr)  \n[http://vision.snu.ac.kr/projects/mpchat](http://vision.snu.ac.kr/projects/mpchat)  \narXiv :2305 . 17388v1 [ cs .CL] 27 May 2023  \nAbstract  \nIn order to build self-consistent personalized dialogue agents, previous research has mostly focused on textual persona that delivers personal facts or personalities. However, to fully describe the multi-faceted nature of persona, image modality can help better reveal the speaker’s personal characteristics and experiences in episodic memory (Rubin et al., 2003 ; Conway, 2009) . In this work, we extend persona-based dialogue to the multimodal domain and make two main contributions. First, we present the first multimodal persona-based dialogue dataset named MPCHAT, which extends persona with both text and images to contain episodic memories. Second, we empirically show that incorporating multimodal persona, as measured by three proposed multimodal persona-grounded dialogue tasks (i.e., next response prediction, grounding persona prediction, and speaker identification), leads to statistically significant performance improvements across all tasks. Thus, our work highlights that multimodal persona is crucial for improving multimodal dialogue comprehension, and our MPCHAT serves as a high-quality resource for this research.  \n1 Introduction  \nWith the rapid advance of conversational AI systems in recent years, developing self-consistent dialogue agents has been studied much (Li et al., 2016 ; Zhang et al., 2018) . Considerable research aims to endow dialogue agents with persona, which represents an individual’s personality (Zhong et al., 2022 ; Cao et al., 2022) . In particular, researchers have exploited textual description of persona, for example, in the form of unstructured sentences (Mazaré et al., 2018), structured key-value attributes (e.g., age, gender, location) (Song et al., 2020) and personality types (e.g., Big-Five) (Mairesse and Walker, 2007) . Therefore, dialogue agents with persona have been found to  \nFigure 1: An example of MPCHAT: a user A’s persona (i.e., five persona image-sentence pairs) in the left and a dialogue example in the right. Each persona element p consists of a pair of an image pi and a sentence pt. Each response from the user A in the dialogue example is grounded on a specific persona element pm . Multimodal personas from MPCHAT describe episodic memories of personal experiences (e.g., favorite rocketsand constellations) with visual details.  \n(1) exhibit greater self-consistency (Welleck et al., 2019 ; Kim et al., 2020 ; Majumder et al., 2020),(2) demonstrate awareness of long-term memory (Xu et al., 2022a,b ; Bae et al., 2022), and (3) generate engaging responses instead of non-specific ones (Zhang et al., 2018 ; Mazaré et al., 2018) .  \nHowever, existing studies restrict the role of persona only to personal facts (Zhang et al., 2018) or personalities (Li et al., 2020a), while it should be explored in multi-faceted ways (Moore et al., 2017) . More than factual information, episodic memory (Tulving, 1972), which is the memory of everyday events or personal experiences connected to the self and autonoetic consciousness (Tulving, 2002 ; Conway, 2005), should be included in persona component. Wilson and Ross (2003) further supports this assertion by arguing that episodic memory plays a significant role in shaping personal  \nidentity, which in turn can influence one’s persona. Since episodic memories are often represented in the form of visual images or history scenes (Rubin et al., 2003 ; Conway, 2009), we propose to study the multimodal persona, which consists of a set of image-sentence pairs describing memorable moments as shown in Figure 1. F","cbCaigczdyDXJ6ps","https://ap.wps.com/l/cbCaigczdyDXJ6ps","pdf",5648280,24,"English","# Abstract\n# Introduction\n# Related Work","[{\"question\":\"What is MPCHAT and what does it contribute to persona-based dialogue research?\",\"answer\":\"MPCHAT is a multimodal persona-based dialogue dataset that extends persona with both text and images to represent episodic memories. It provides grounded multimodal dialogue examples for research on multimodal persona in conversation.\"},{\"question\":\"How does the paper define multimodal persona in this work?\",\"answer\":\"Multimodal persona is formulated as a set of image-sentence pairs that describe memorable moments. Each user response in the dialogue is grounded on a corresponding persona element.\"},{\"question\":\"What benchmark tasks are proposed for multimodal persona-grounded dialogue understanding?\",\"answer\":\"The paper proposes three retrieval-based dialogue tasks: next response prediction, grounding persona prediction, and speaker identification. Incorporating multimodal persona yields statistically significant performance gains across all tasks.\"}]","MPCHAT - Towards Multimodal Persona-Grounded Conversation - Abstract and Introduction | PDF",1792773533]