[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-293662-105":59,"doc-detail-293662-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","incharacter-evaluating-personality-fidelity-in-role-playing-agents-through-psychological-interviews","INCHARACTER - Evaluating Personality Fidelity in Role-Playing Agents - Through Psychological Interviews","","Role-playing agents (RPAs) powered by large language models face a key challenge: reproducing the target characters’ personas with accurate personality fidelity. The work introduces INCHARACTER, an interview-based method that evaluates personality fidelity using psychological scales. Instead of self-report options that may conflict with role-playing instructions or distort results, INCHARACTER elicits mindsets and behaviors via open-ended interviews and applies LLM-based assessment to measure alignment.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/incharacter-evaluating-personality-fidelity-in-role-playing-agents-through-psychological-interviews/293662/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/incharacter-evaluating-personality-fidelity-in-role-playing-agents-through-psychological-interviews/293662.png","ImageObject",300,407,{"name":92,"@type":93},"Fez","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-11","2026-09-17",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",14,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What problem does INCHARACTER address in role-playing agents?","Question",{"text":113,"@type":114},"It addresses how to evaluate whether RPAs accurately reproduce the target characters’ personas, focusing on personality fidelity rather than only knowledge or linguistic patterns.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"How does INCHARACTER differ from self-report personality scales?",{"text":118,"@type":114},"INCHARACTER uses interview-based, open-ended questions derived from psychological scales and then uses LLMs to interpret responses, avoiding conflicts between scale instructions and role-playing behavior.",{"name":120,"@type":111,"acceptedAnswer":121},"What do the experiments evaluate and measure?",{"text":122,"@type":114},"The experiments apply INCHARACTER across multiple RPAs and LLMs, covering 32 distinct characters on 14 personality scales and measuring how closely RPA personalities align with human-perceived character personalities.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},293662,1789667301,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":130,"read_time":145,"preview_page_count":4},2336478940794,"https://ap-avatar.wpscdn.com/davatar_9964176cb1d06d4a9deccf72a44ae3dc","INCHARACTER: Evaluating Personality Fidelity in Role-Playing Agents  \nthrough Psychological Interviews  \nXintao Wang 1 , Yunze Xiao2 , Jen-tse Huang3 , Siyu Yuan4 , Rui Xu 1 , Haoran Guo5 , Quan Tu6 , Yaying Fei7 , Ziang Leng8 , Wei Wang 1 , Jiangjie Chen 1 , Cheng Li9 , Yanghua Xiao∗1  \n1 Shanghai Key Laboratory of Data Science, School of Computer Science, Fudan University  \n2 Carnegie Mellon University 3The Chinese University of Hong Kong  \n4 School of Data Science, Fudan University  \n7Beijing University of Technology  \n5RhineAI 6Renmin University of China  \n8Boston University 9 SenseTime  \n{xtwang21, syyuan21, [ruixu21}@m.fudan.edu.cn](ruixu21}@m.fudan.edu.cn) ,{jjchen19, weiwang1, [shawyh}@fudan.edu.cn](shawyh}@fudan.edu.cn)  \n[yunzex@andrew.cmu.edu](yunzex@andrew.cmu.edu) , [jthuang@cse.cuhk.edu.hk](jthuang@cse.cuhk.edu.hk) , [chengli@sensetime.com](chengli@sensetime.com)  \nAbstract  \nRole-playing agents (RPAs), powered by large language models, have emerged as a flourishing field of applications. However, a key challenge lies in assessing whether RPAs accurately reproduce the personas of target characters, namely their character fidelity. Existing methods mainly focus on the knowledge and linguistic patterns of characters. This paper, instead, introduces a novel perspective to evaluate the personality fidelity of RPAs with psychological scales. Overcoming drawbacks of previous self-report assessments on RPAs, we propose INCHARACTER, namely Interviewing Character agents for personality tests. Experiments include various types of RPAs and LLMs, covering 32 distinct characters on 14 widely used psychological scales. The results validate the effectiveness of INCHARACTER in measuring RPA personalities. Then, with INCHARACTER, we show that state-of-the-art RPAs exhibit personalities highly aligned with the human-perceived personalities of the characters, achieving an accuracy up to 80 .7% .1  \n1 Introduction  \nRecent advancements in large language models (LLMs) have catalyzed the emergence of roleplaying agents (RPAs) . RPAs are interactive AI systems simulating diverse roles or characters. RPA applications have been extended to diverse contexts, such as AI agents of fictional characters (Li et al., 2023), digital clones for humans (Gao et al., 2023), and AI non-player characters in videogames (Wang et al., 2023a) . Recent research trends have increasingly focused on the development of  \n*  \nCorresponding author.  \n1Project page: [https://incharacter.github.io/](https://incharacter.github.io/)  \nFigure 1: The procedure of personality tests on RPAs. To evaluate the personality fidelity of RPAs, we apply various scales to measure their personalities and compare the results with the personality labels of the characters.  \nRPAs, including building RPAs for specific characters (Li et al., 2023 ; Wang et al., 2023b) and improving the role-playing abilities of foundation models (Zhou et al., 2023) .  \nHowever, evaluating character fidelity in RPAs remains a relatively underexplored area. Prior research mainly concentrates on the replication of knowledge, experience, and linguistic patterns of characters (Shao et al., 2023a ; Zhou et al., 2023), which manifests in two primary issues: (1) They necessitate character-specific datasets, thereby complicating the evaluation of new characters. (2) They overlook evaluating RPAs’ thoughts and underlying mindsets. Towards these issues, we propose to evaluate if RPAs faithfully reproduce the personalities of target characters, i.e., personality fidelity, as depicted in Figure 1. Personality tests, administered by psychological scales, measure an individual’s interrelated behavioral, cognitive, and emotional patterns (Barrick and Mount, 1991 ; Bem, 1981) . By measuring the personalities of RPAs and com-  \n1840  \nProceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) , pages 1840–1873  \nAugust 11-16, 2024 ©2024 Association for Computational ","cbCaivHgJi2XmUSE","https://ap.wps.com/l/cbCaivHgJi2XmUSE","pdf",2824291,34,"English","# Abstract\n# Introduction\n## Role-playing agents and the fidelity challenge\n## Personality testing with psychological scales\n## Limitations of self-report methods\n## INCHARACTER: interview-based evaluation","[{\"question\":\"What problem does INCHARACTER address in role-playing agents?\",\"answer\":\"It addresses how to evaluate whether RPAs accurately reproduce the target characters’ personas, focusing on personality fidelity rather than only knowledge or linguistic patterns.\"},{\"question\":\"How does INCHARACTER differ from self-report personality scales?\",\"answer\":\"INCHARACTER uses interview-based, open-ended questions derived from psychological scales and then uses LLMs to interpret responses, avoiding conflicts between scale instructions and role-playing behavior.\"},{\"question\":\"What do the experiments evaluate and measure?\",\"answer\":\"The experiments apply INCHARACTER across multiple RPAs and LLMs, covering 32 distinct characters on 14 personality scales and measuring how closely RPA personalities align with human-perceived character personalities.\"}]","INCHARACTER - Evaluating Personality Fidelity in Role-Playing Agents - Through Psychological Interviews | PDF",86]