[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-201213-105":59,"doc-detail-201213-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","folio-natural-language-reasoning-with-first-order-logic","FOLIO - Natural Language Reasoning with First-Order Logic","","Large language models (LLMs) achieve strong results on many language understanding tasks, yet existing benchmarks inadequately measure complex logical reasoning ability. This work introduces FOLIO, a human-annotated dataset for natural language (NL) reasoning, equipped with first-order logic (FOL) annotations. FOLIO contains 1,430 examples with 487 premise sets and validates logical correctness via an FOL inference engine. It also provides NL-FOL translation pairs and evaluates fine-tuning on medium-sized models across multiple state-of-the-art systems.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/folio-natural-language-reasoning-with-first-order-logic/201213/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/folio-natural-language-reasoning-with-first-order-logic/201213.png","ImageObject",300,407,{"name":92,"@type":93},"Emma Mercer","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-28","2026-09-04",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",17,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What problem does FOLIO address in evaluating LLM reasoning?","Question",{"text":113,"@type":114},"Existing natural language benchmarks do not adequately measure complex logical reasoning capabilities. FOLIO focuses on logical deduction correctness grounded in premises and conclusions annotated with first-order logic.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"How is logical correctness guaranteed in FOLIO?",{"text":118,"@type":114},"Each NL reasoning instance is paired with FOL annotations that are automatically verified by an FOL inference engine to ensure premise and conclusion validity.",{"name":120,"@type":111,"acceptedAnswer":121},"What additional task does FOLIO support beyond NL reasoning?",{"text":122,"@type":114},"FOLIO also provides NL-FOL translation pairs, enabling evaluation and benchmarking for translation between natural language and first-order logic.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},201213,1788527357,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":130,"read_time":145,"preview_page_count":4},962084925502,"https://ap-avatar.wpscdn.com/davatar_6f874abed73319feea01a86fa6f0fab8","FOLIO: Natural Language Reasoning with First-Order Logic  \nSimeng Han 1 Hailey Schoelkopf1 Yilun Zhao 1 Zhenting Qi2 Martin Riddell 1 Wenfei Zhou3 James Coady 1 David Peng 1 Yujie Qiao 1 Luke Benson 1 Lucy Sun 1 Alex Wardle-Solano 1 Hannah Szabo 1 Ekaterina Zubova 1 Matthew Burtell 1 Jonathan Fan4 Yixin Liu 1 Brian Wong 1 Malcolm Sailor 1 Ansong Ni 1 Linyong Nan 1 Jungo Kasai5 Tao Yu6 Rui Zhang7  \nAlexander R. Fabbri9 Wojciech Kryciski9  \nSemih Yavuz9 Ye Liu9 Xi Victoria Lin8 Shafiq Joty9 Yingbo Zhou9 Caiming Xiong9 Rex Ying 1 Arman Cohan 1 Dragomir Radev 1 ,9  \n1 Yale University, 2Harvard University, 3NVIDIA, 4Iowa City West High School  \n5University of Washington, 6University of Hong Kong  \n7Penn State University, 8Meta AI, 9 Salesforce Research  \nAbstract  \nLarge language models (LLMs) have achieved remarkable performance on a variety of natural language understanding tasks. However, existing benchmarks are inadequate in measuring the complex logical reasoning capabilities of a model. We present FOLIO, a human-annotated, logically complex and diverse dataset for reasoning in natural language (NL), equipped with first-order logic (FOL) annotations. FOLIO consists of 1,430 examples (unique conclusions), each paired with one of 487 sets of premises used to deductively reason for the validity of each conclusion. The logical correctness of the premises and conclusions is ensured by their FOL annotations, which are automatically verified by an FOL inference engine. In addition to the main NL reasoning task, NLFOL pairs in FOLIO constitute a new NL-FOL translation dataset. Our experiments on FOLIO systematically evaluate the FOL reasoning ability of supervised fine-tuning on medium-sized language models. For both NL reasoning and NL-FOL translation, we benchmark multiple state-of-the-art language models. Our results show that a subset of FOLIO presents a challenge for one of the most capable Large Language Model (LLM) publicly available, GPT-4 .  \n1 Introduction  \nLarge language models (LLMs) have achieved remarkable performance on a variety of natural language tasks (OpenAI et al., 2023 ; Touvron et al., 2023 ; Srivastava et al., 2023 ; Wang et al., 2019a) . Logical reasoning is a central component for intelligent systems and should be sufficiently and independently evaluated (Russell and Norvig, 2010) .  \nHowever, existing natural language tasks are inadequate in measuring the complex logical reasoning capability of a model (Srivastava et al., 2023 ; Saparov and He, 2023 ; Tian et al., 2021) .  \nSeveral datasets related to logical reasoning have recently been proposed. However, existing benchmarks often exhibit limited complexity in reasoning or lack language naturalness. Some of these common benchmarks do not specifically evaluate logical reasoning independently of other forms of reasoning (Yu et al., 2020 ; Liu et al., 2021) . Those specifically designed for measuring logical reasoning are insufficient in terms of logical reasoning complexity and natural language variety. As shown in Table 1, examples in RuleTaker (Clark et al., 2020) and LogicNLI (Tian et al., 2021) need at most five depths of reasoning. The entire corpus of RuleTaker or LogicNLI has fewer than 50 distinct abstract syntax trees. RuleTaker has only 101 words in its vocabulary and LogicNLI has 1077 words in the vocabulary. Moreover, none of them are written by humans with information drawn from real-world knowledge, making them less applicable to real-world reasoning scenarios. The logical deduction portion in BigBench (Srivastava et al., 2023) requires commonsense reasoning besides logical deduction. ProntoQA (Saparov and He, 2023) only contains logical reasoning questions that are answerable with repeated applications of the Modus Ponens inference rule.  \nWe present a natural language reasoning dataset, FOLIO, with first-order logic reasoning problems which require the models to decide the correctness of conclusions given a world defined by the premises. In F","cbCair1zzhwdsJRe","https://ap.wps.com/l/cbCair1zzhwdsJRe","pdf",385584,15,"English","# Abstract\n# 1 Introduction\n## Background and limitations of existing benchmarks\n## The FOLIO dataset and its design\n## Experimental evaluation and benchmarking","[{\"question\":\"What problem does FOLIO address in evaluating LLM reasoning?\",\"answer\":\"Existing natural language benchmarks do not adequately measure complex logical reasoning capabilities. FOLIO focuses on logical deduction correctness grounded in premises and conclusions annotated with first-order logic.\"},{\"question\":\"How is logical correctness guaranteed in FOLIO?\",\"answer\":\"Each NL reasoning instance is paired with FOL annotations that are automatically verified by an FOL inference engine to ensure premise and conclusion validity.\"},{\"question\":\"What additional task does FOLIO support beyond NL reasoning?\",\"answer\":\"FOLIO also provides NL-FOL translation pairs, enabling evaluation and benchmarking for translation between natural language and first-order logic.\"}]","FOLIO - Natural Language Reasoning with First-Order Logic | PDF",38]