[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-516153-105":59,"doc-detail-516153-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","anatom-a-dataset-generation-framework-for-evaluating-theory-of-mind-reasoning-toward-anatomy-of-difficulty-through-structurally-controlled-story-generation","AnaToM: A Dataset Generation Framework for Evaluating Theory of Mind Reasoning Toward Anatomy of Difficulty Through Structurally Controlled Story Generation","","AnaToM is a dataset generation framework for evaluating Theory of Mind (ToM) reasoning in large language models by systematically controlling the structural parameters that determine task difficulty. By strictly manipulating factors such as the number of entities and the story timeline, it isolates and identifies which elements affect ToM-related reasoning, while controlling semantic confounds via template-based synthetic data. The framework supports diagnosing structural limits of LLM reasoning and provides a structural diagnostic baseline to complement future benchmark designs.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/anatom-a-dataset-generation-framework-for-evaluating-theory-of-mind-reasoning-toward-anatomy-of-difficulty-through-structurally-controlled-story-generation/516153/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/anatom-a-dataset-generation-framework-for-evaluating-theory-of-mind-reasoning-toward-anatomy-of-difficulty-through-structurally-controlled-story-generation/516153.png","ImageObject",300,407,{"name":92,"@type":93},"BAd apple","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-16","2026-10-06",true,{"@type":102,"interactionType":103,"userInteractionCount":24},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What problem does AnaToM address in evaluating Theory of Mind (ToM) reasoning for LLMs?","Question",{"text":112,"@type":113},"Existing ToM benchmarks often make tasks harder or more comprehensive, but they do not provide a systematic analysis of structural factors that inherently determine ToM reasoning difficulty. AnaToM aims to isolate these factors.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does AnaToM control task difficulty in its generated stories?",{"text":117,"@type":113},"AnaToM strictly controls structural parameters such as the number of entities and the timeline in a story, enabling separation of elements that drive reasoning difficulty.",{"name":119,"@type":110,"acceptedAnswer":120},"Why does AnaToM use template-based synthetic data generation?",{"text":121,"@type":113},"It intentionally controls semantic variables (e.g., linguistic diversity) so confounding effects on reasoning difficulty are reduced, improving the diagnosis of structural limits.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},516153,1791821622,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":24,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":145,"preview_page_count":19,"preview_urls":146},19241458193282,"https://us-avatar.wpscdn.com/davatar_a8503ba1806abce46bf441b54a3ca4cd","General Paper  \nPeer-Reviewed  \nAnaToM: A Dataset Generation Framework for Evaluating Theory of Mind Reasoning Toward Anatomy of Diﬃculty Through Structurally Controlled Story Generation  \nJundai Suzukia , Ryoma Ishigakia and Eisaku Maedaa  \nEvaluating Theory of Mind (ToM) in Large Language Models (LLMs) is an important area of research for understanding the social intelligence of artiﬁcial intelligence. Recent ToM benchmarks have signiﬁcantly enhanced the complexity, comprehensiveness, and practicality of evaluations. However, while the focus has been on constructing“more diﬃcult” or “more comprehensive” tasks, systematic analysis of structural factors that inherently determine the diﬃculty of ToM reasoning, i.e., “what” makes reasoning diﬃcult, is insuﬃcient. Hence, we propose a new dataset generation framework for ToM evaluation, named AnaToM. To realize an “anatomy of diﬃculty” in ToM reasoning, AnaToM strictly controls structural parameters such as the number of entities and the timeline in a story. This parameter control enables the isolation and identiﬁcation of factors aﬀecting ToM in LLMs, thereby enabling a more precise examination of their reasoning mechanisms. The proposed framework provides a systematic methodology for diagnosing the structural limits of LLM reasoning abilities, thus oﬀering a foundational diagnostic baseline that complements the evaluation of broader sociocognitive capabilities in future benchmark designs.  \nKey Words: Theory of Mind, Large Language Models, Evaluation, Benchmark, Social Reasoning, Cognitive Modeling, Controlled Story Generation  \n1 Introduction  \nAs artiﬁcial intelligence (AI) agents become increasingly prevalent in society, their social intelligence, particularly their ability to interact and collaborate smoothly with humans, is becoming increasingly importance. At the core of this social intelligence is the ability to infer mental states such as intentions and beliefs from others’words and actions, known as “Theory of Mind”(ToM)(Premack and Woodruﬀ 1978) . ToM has been extensively investigated in cognitive science, and  \na Tokyo Denki University  \nThis paper is based on [https://aclanthology.org/2025.findings-ijcnlp.14/](https://aclanthology.org/2025.findings-ijcnlp.14/) , which was accepted to the Findings of the Association for Computational Linguistics: IJCNLP-AACL 2025 (Suzuki et al. 2025) .  \n(C) The Association for Natural Language Processing. Licensed under CC BY 4.0 ([https://creativecommons.org/licenses/by/4.0/](https://creativecommons.org/licenses/by/4.0/)).  \nfalse belief tasks, in particular, are integral in understanding human ToM development and its neurological basis (Wimmer and Perner 1983; Baron-Cohen et al. 1985) .  \nIn recent years, this classical framework has been extended to evaluate the capabilities of Large Language Models (LLMs) . Whether LLMs possess ToM is actively debated (Sap et al. 2022; Ullman 2023; Ma et al. 2023; Sarıtaş et al. 2025), and methods for evaluating ToM have evolved rapidly. An early benchmark, ToMi (Le et al. 2019), highlighted the problem of pattern learning in template-based tasks, thus providing a foundation for subsequent research. Hence, recent studies have pursued reﬁnement through multifaceted approaches, including comprehensiveness (OpenToM (Xu et al. 2024), ToMBench (Chen et al. 2024)), higher-order reasoning (Hi-ToM (Wu et al. 2023)), conversational formats (ToMATO (Shinoda et al. 2025)), real-world data (CommonToM (Soubki et al. 2024)), and adversarial data generation by LLMs (ExploreToM (Sclar et al. 2025), BigToM (Gandhi et al. 2023)) .  \nThese studies have contributed signiﬁcantly to the multifaceted evaluation of ToM in LLMs. However, they focus primarily on constructing “more diﬃcult” or “more comprehensive” tasks, and systematic analysis of the structural factors that inherently determine the diﬃculty of ToM reasoning, i.e., what renders the reasoning diﬃcult, remains insuﬃcient. Consequently, factors that aﬀect the","cbCaiq60qWKM242n","https://ap.wps.com/l/cbCaiq60qWKM242n","pdf",400445,28,"English","# Introduction\n## Motivation and background on ToM evaluation in LLMs\n## Limitations of existing benchmarks\n## AnaToM framework overview and goals","[{\"question\":\"What problem does AnaToM address in evaluating Theory of Mind (ToM) reasoning for LLMs?\",\"answer\":\"Existing ToM benchmarks often make tasks harder or more comprehensive, but they do not provide a systematic analysis of structural factors that inherently determine ToM reasoning difficulty. AnaToM aims to isolate these factors.\"},{\"question\":\"How does AnaToM control task difficulty in its generated stories?\",\"answer\":\"AnaToM strictly controls structural parameters such as the number of entities and the timeline in a story, enabling separation of elements that drive reasoning difficulty.\"},{\"question\":\"Why does AnaToM use template-based synthetic data generation?\",\"answer\":\"It intentionally controls semantic variables (e.g., linguistic diversity) so confounding effects on reasoning difficulty are reduced, improving the diagnosis of structural limits.\"}]","AnaToM: A Dataset Generation Framework for Evaluating Theory of Mind Reasoning Toward Anatomy of Difficulty Through Structurally Controlled Story Generation | PDF",1791326622,71,[147,151,155,159],{"page_no":8,"url":148,"width":149,"height":150},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/f8cb1b5716001d47eba50237a53e9dd0.webp",983,1387,{"page_no":14,"url":152,"width":153,"height":154},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/d4c7a3efa53d6bd262322c69eaffd44e.webp",982,1386,{"page_no":81,"url":156,"width":157,"height":158},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/a5f2ad125ec40514ed1247e7d2e94856.webp",981,1385,{"page_no":19,"url":160,"width":161,"height":162},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/fb6d128b12530db5298923996c22a845.webp",980,1383]