[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-291229-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-291229-en":131},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","evaluating-numeracy-of-language-models-as-a-natural-language-inference-task-paper","Evaluating Numeracy of Language Models as a Natural Language Inference Task - paper","","Evaluating numeracy in language models is treated as a Natural Language Inference benchmark targeting both numerical understanding and linguistic context. The study introduces a benchmark for basic numeracy tasks and frames them as entailment-based reasoning over numbers and narratives. Results from 49 language models, including fine-tuned NLI LMs, instruction-tuned LLMs, and specialized math-LLMs, show strong but limited arithmetic gains, persistent difficulty with contradiction/neutral predictions, and the need for robustness in numerals’ semantics and pragmatics.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/evaluating-numeracy-of-language-models-as-a-natural-language-inference-task-paper/291229/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/evaluating-numeracy-of-language-models-as-a-natural-language-inference-task-paper/291229.png","ImageObject",300,407,{"name":42,"@type":43},"Kyle","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-10-28","2026-09-17",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",12,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"How is numeracy evaluated in this paper?","Question",{"text":63,"@type":64},"Numeracy is framed as a Natural Language Inference (NLI) task, assessing how models use numerical information within textual context to draw inferences or conclusions.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"What numeracy skills are tested beyond arithmetic?",{"text":68,"@type":64},"The evaluation covers arithmetic reasoning plus number comparison and number normalization across different syntactic formats (e.g., “three” vs “3”).",{"name":70,"@type":61,"acceptedAnswer":71},"What are the main findings about model performance?",{"text":72,"@type":64},"LLMs clearly outperform smaller models mainly for arithmetic tasks, but they still struggle with contradiction and neutral cases; robustness is also limited in understanding numerical semantics and pragmatics in language contexts.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},291229,1789643811,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,111,115,120,123,127],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":108,"show_sort_weight":109,"slug":110},7,"Healthcare",40,"healthcare",{"id":112,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":113,"slug":114},8,30,"research-report",{"id":116,"doc_module":4,"doc_module_name":25,"category_name":117,"show_sort_weight":118,"slug":119},9,"Religion & Spirituality",20,"religion-spirituality",{"id":118,"doc_module":4,"doc_module_name":25,"category_name":121,"show_sort_weight":118,"slug":122},"World Cup","world-cup",{"id":124,"doc_module":4,"doc_module_name":25,"category_name":125,"show_sort_weight":124,"slug":126},10,"Lifestyle","lifestyle",{"id":128,"doc_module":4,"doc_module_name":25,"category_name":129,"show_sort_weight":97,"slug":130},19,"General","general",{"code":4,"msg":82,"data":132},{"doc_id":79,"user_id":133,"nickname":42,"user_avatar":134,"doc_module":4,"category_id":112,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":140,"language":141,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":12,"update_tm":80,"read_time":145,"preview_page_count":4},3985741905716,"https://ap-avatar.wpscdn.com/davatar_994ba38a5ba835b3df7d355c54d3ed8d","Evaluating Numeracy of Language Models as a Natural Language Inference Task  \nRahmad Mahendra1 , Damiano Spina1 , Lawrence Cavedon1 , Karin Verspoor1,2  \n1RMIT University, School of Computing Technologies  \n2The University of Melbourne, School of Computing and Information Systems Melbourne, Australia  \n[rahmad.mahendra@student.rmit.edu.au](rahmad.mahendra@student.rmit.edu.au)  \n{damiano.spina,lawrence.cavedon, [karin.verspoor}@rmit.edu.au](karin.verspoor}@rmit.edu.au)  \nAbstract  \nWhile recent advancements in large language models (LLMs) have enhanced their capabilities to solve mathematical problems, other aspects of numeracy remain underexplored. In this paper, we propose a benchmark to evaluate the ability of language models to perform basic numeracy tasks. We frame numeracy as a Natural Language Inference (NLI) task to assess the models’ ability to understand both numbers and language contexts. We evaluate 49 language models (LMs), including fine-tuned LMs on NLI datasets, instruction-tuned LLMs, and specialized math-LLMs. Our findings reveal three main insights: (1) LLMs only clearly outperform smaller LMs in arithmetic tasks, indicating that mathematical reasoning cannot be generalized to other numeracy skills such as number comparison and normalization; (2) while most language models achieve fair to good accuracy for NLI entailment cases, they still struggle to predict contradiction and neutral cases;  \nand (3) the robustness of language models’numeracy capabilities needs improvement, particularly in understanding the semantics and pragmatics of numbers in linguistic contexts.  \n1 Introduction  \nLanguage models (LMs) have made significant strides in the field of natural language processing (NLP) and proven to be highly adaptable toa diverse array of language-related tasks. Current benchmarks for LMs on numerical reasoning predominantly focus on answering math questions derived from test material, such as standardized assessments (Hendrycks et al., 2021 ; Cobbe et al., 2021 ; Liu et al., 2024 ; He et al., 2024) . Recent results show remarkable progress of very large language models to decipher complex mathematics problems, such as word problems with multiple unknown variables and non-linear equations (Lu et al., 2023 ; Ahn et al., 2024) . While solving mathematical challenges gain increasing attention in  \nthe research community, the application of numerical reasoning to real-world scenarios remains relatively unexplored. In such contexts, LLMs must not only be able to answer numerical questions but also verify the correctness of natural language text containing numerical information. It is important to expand the evaluation paradigm to include more application-oriented tasks where numerical reasoning plays a central role. For instance, fact-checking involves verifying numerical claims from textual data (Thorne and Vlachos, 2017 ; Venktesh et al., 2024) or scientific hypothesis testing requires numerical interpretation 1 (Lehman et al., 2019) . Successful numerical reasoning in such tasks requires not only mathematical problem solving, but also a comprehension of numerical concepts (Wallace et al., 2019 ; Thawani et al., 2021a) .  \nWe investigate the performance of language models on foundational numeracy tasks. We test not only arithmetic reasoning but also two other essential numeracy skills, i.e., comparing numbers and understanding a number when expressed in different syntactic formats (e.g.,“three” vs “3”) . Our evaluation adapts natural language inference (NLI) tasks to assess the numeracy capability of language models. We analyze how well a model can use numerical information in context to make inferences or draw conclusions. Models are evaluated on their ability to disambiguate and reason about potentially conflicting information in text. NLI tasks need to deal with ambiguity, which is common when working with numerical information. Moreover, NLI tasks offer a broader evaluation of a model’s overall language comp","cbCairXW5RXU7rcB","https://ap.wps.com/l/cbCairXW5RXU7rcB","pdf",1292516,26,"English","# Abstract\n# 1 Introduction\n## Foundational numeracy tasks and NLI framing\n# Table 1: Examples of numeracy premise-hypothesis pairs","[{\"question\":\"How is numeracy evaluated in this paper?\",\"answer\":\"Numeracy is framed as a Natural Language Inference (NLI) task, assessing how models use numerical information within textual context to draw inferences or conclusions.\"},{\"question\":\"What numeracy skills are tested beyond arithmetic?\",\"answer\":\"The evaluation covers arithmetic reasoning plus number comparison and number normalization across different syntactic formats (e.g., “three” vs “3”).\"},{\"question\":\"What are the main findings about model performance?\",\"answer\":\"LLMs clearly outperform smaller models mainly for arithmetic tasks, but they still struggle with contradiction and neutral cases; robustness is also limited in understanding numerical semantics and pragmatics in language contexts.\"}]","Evaluating Numeracy of Language Models as a Natural Language Inference Task - paper | PDF",66]