[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-327806-105":59,"doc-detail-327806-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","consistent-but-miscalibrated-evaluating-llm-limitations-for-risk-communication-in-natural-language-327806","Consistent but Miscalibrated - Evaluating LLM Limitations for Risk Communication in Natural Language","","LLMs are increasingly used as post-hoc explainers, yet it is unclear whether they reliably communicate probabilistic information in natural language. This study evaluates nine LLMs within a two-stage pipeline where an upstream model generates probabilistic outputs with likelihood and uncertainty, and LLMs select verbal descriptors. Using simulated Beta-distribution predictions across six contexts and ten temperatures, repeated ten times, results show general consistency but systematic miscalibration, especially for uncertainty.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/consistent-but-miscalibrated-evaluating-llm-limitations-for-risk-communication-in-natural-language-327806/327806/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/consistent-but-miscalibrated-evaluating-llm-limitations-for-risk-communication-in-natural-language-327806/327806.png","ImageObject",300,407,{"name":92,"@type":93},"Asher","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-10","2026-09-21",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",11,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What problem does the study address about LLMs in risk communication?","Question",{"text":113,"@type":114},"It tests whether LLMs can produce consistent and well-calibrated natural-language descriptors for probabilistic predictions, especially when communicating uncertainty.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"How is calibration and consistency evaluated in this work?",{"text":118,"@type":114},"Models are prompted to verbalize likelihood and uncertainty descriptors for simulated predictive distributions, then assessed using metrics for whether repeated identical inputs yield identical descriptors (consistency) and whether descriptor strength matches known quantities (calibration).",{"name":120,"@type":111,"acceptedAnswer":121},"What were the main findings regarding likelihood versus uncertainty?",{"text":122,"@type":114},"LLMs were generally consistent, but miscalibrated; performance was substantially weaker on uncertainty tasks than on likelihood tasks, indicating the verbalization step limits reliable risk communication.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},327806,1790527976,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":145,"read_time":146,"preview_page_count":4},687197207639,"https://ap-avatar.wpscdn.com/davatar_a8503ba1806abce46bf441b54a3ca4cd","arXiv :2607 .03882v 1 [ cs .CL] 4 Jul 2026  \nConsistent but Miscalibrated: Evaluating LLM Limitations for Risk Communication in Natural Language  \nDiego Cerda-Mardini1,2,3 , Sarath Chandar2,3,4,5 , and Sreenath Madathil1,3  \n1Faculty of Dental Medicine and Oral Health Sciences, McGill University, Montréal, QC, Canada, 2 Chandar Research Lab, Polytechnique Montréal, Montréal, H3T 1J4, Canada, 3 Mila – Québec Artificial Intelligence Institute, Montréal, H2S 3H1, Canada, 4 Département de Génie Informatique et Génie Logiciel (GIGL), Polytechnique Montréal, Montréal, H3T 1J4, Canada, 5 Canada CIFAR AI Chair  \nLLMs are increasingly deployed as post-hoc explainers of AI-generated outputs, yet it remains unclear whether they can reliably communicate probabilistic information in natural language. For this role to be viable, models must produce identical verbal descriptions for identical inputs, and select descriptions that accurately reflect the magnitude of the underlying numerical quantities. We evaluate whether nine LLMs meet these requirements within a two-stage prediction pipeline, in which an upstream model has produced probabilistic outputs characterized by their likelihood and uncertainty, and LLMs are tasked with selecting an appropriate verbal descriptor for each. We simulate predictions from an upstream model by taking samples from a Beta distribution parameterized by its mode and prior sample size. We then prompt LLMs to explain these predictions under six domain contexts and with ten temperature settings, and repeating each experiment ten times. We find that LLMs are generally consistent but miscalibrated, with substantially weaker performance on uncertainty than on likelihood tasks. Providing models with precomputed summary statistics (mode and prior sample size) reduced sensitivity to contextual framing but did not resolve the underlying miscalibration, suggesting that the bottleneck resides in the verbalization step itself. These findings indicate that current LLMs do not yet constitute reliable zero-shot standalone risk communication tools for probabilistic predictions.  \n Code Repository: github/CTruAI/risk_communication_llm  \n1. Introduction  \nLarge Language Models (LLMs) are increasingly used in high-stakes domains, such as healthcare, finance and law, where the cost of incorrect predictions can be severe [Maity and Jyoti, 2025, Dehghani et al., 2025, Nie et al., 2024] . In such settings, it is not enough for a model to be accurate; users also need to knowhow confident the model is in its predictions [Band et al., 2024] . Calibrated uncertainty scores address this need by indicating when to trust a model’s prediction [Minderer et al., 2021, Madsen et al., 2024] . Yet even when such scores are available, correctly interpreting them poses a separate challenge for end users. Concepts like “statistical uncertainty” are frequently misinterpreted, even by skilled professionals like doctors [Gigerenzer et al., 2007] . Predictive distributions can be characterized by two properties: the likelihood of an event occurring and the uncertainty in its prediction [Tyralis and Papacharalampous, 2024] . How laypeople intuitively understand probability often di-  \nverges from its formal statistical definition [Hashim, 2024], adding to the misunderstanding of risk. As reliance on AI for decision-making grows, there is need to convey predictive concepts like likelihood and uncertainty to audiences with varying statistical literacy. LLMs are a promising vehicle for this, as they maybe able to translate numerical estimates into Natural Language Explanations (NLEs) that are more interpretable to non-expert users [Kayser et al., 2022, 2021, Stern et al., 2024] . This raises two questions: whether LLMs have a coherent internal mapping of concepts like likelihood and uncertainty, and whether they can express these concepts in language that is consistent and calibrated to the underlying probabilistic information. To address this, we evaluated","cbCaik6TAm7o18n2","https://ap.wps.com/l/cbCaik6TAm7o18n2","pdf",5744917,23,"English","# Introduction\n## Definitions\n## Related works\n### The challenge of risk communication","[{\"question\":\"What problem does the study address about LLMs in risk communication?\",\"answer\":\"It tests whether LLMs can produce consistent and well-calibrated natural-language descriptors for probabilistic predictions, especially when communicating uncertainty.\"},{\"question\":\"How is calibration and consistency evaluated in this work?\",\"answer\":\"Models are prompted to verbalize likelihood and uncertainty descriptors for simulated predictive distributions, then assessed using metrics for whether repeated identical inputs yield identical descriptors (consistency) and whether descriptor strength matches known quantities (calibration).\"},{\"question\":\"What were the main findings regarding likelihood versus uncertainty?\",\"answer\":\"LLMs were generally consistent, but miscalibrated; performance was substantially weaker on uncertainty tasks than on likelihood tasks, indicating the verbalization step limits reliable risk communication.\"}]","Consistent but Miscalibrated - Evaluating LLM Limitations for Risk Communication in Natural Language | PDF",1789978099,58]