[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-273550-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-273550-en":131},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","the-iol-ai-challenge-an-open-challenge-towards-advancing-linguistic-reasoning","The IOL-AI Challenge - An Open Challenge towards Advancing Linguistic Reasoning","","Reasoning in large language models is often studied in rule-driven settings like math and code, where the system is known. Linguistic puzzles invert this by requiring solvers to first infer the underlying structure from limited evidence, then reason within it. The IOL-AI Challenge is an open-science competition on unseen International Linguistics Olympiad (IOL) 2026 Individual Contest problems, judged automatically and by the official IOL Jury.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/the-iol-ai-challenge-an-open-challenge-towards-advancing-linguistic-reasoning/273550/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/the-iol-ai-challenge-an-open-challenge-towards-advancing-linguistic-reasoning/273550.png","ImageObject",300,407,{"name":42,"@type":43},"Maeve","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-10-30","2026-09-15",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",11,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What makes the IOL-AI Challenge different from typical reasoning benchmarks?","Question",{"text":63,"@type":64},"It focuses on linguistic puzzles where the solver must infer the underlying system from evidence before reasoning, rather than being given explicit rules up front.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How is the competition evaluated?",{"text":68,"@type":64},"Solutions are evaluated automatically and, for the first time, also by members of the official IOL Jury using the same rubrics as human contestants.",{"name":70,"@type":61,"acceptedAnswer":71},"What were the main findings about model capability?",{"text":72,"@type":64},"Capability was not determined by scale: smaller submissions could outperform larger ones, and automatic metrics agreed with jury rankings while compressing the score range.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},273550,1789472018,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,111,115,120,123,127],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":108,"show_sort_weight":109,"slug":110},7,"Healthcare",40,"healthcare",{"id":112,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":113,"slug":114},8,30,"research-report",{"id":116,"doc_module":4,"doc_module_name":25,"category_name":117,"show_sort_weight":118,"slug":119},9,"Religion & Spirituality",20,"religion-spirituality",{"id":118,"doc_module":4,"doc_module_name":25,"category_name":121,"show_sort_weight":118,"slug":122},"World Cup","world-cup",{"id":124,"doc_module":4,"doc_module_name":25,"category_name":125,"show_sort_weight":124,"slug":126},10,"Lifestyle","lifestyle",{"id":128,"doc_module":4,"doc_module_name":25,"category_name":129,"show_sort_weight":97,"slug":130},19,"General","general",{"code":4,"msg":82,"data":132},{"doc_id":79,"user_id":133,"nickname":42,"user_avatar":134,"doc_module":4,"category_id":112,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":140,"language":141,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":12,"update_tm":80,"read_time":145,"preview_page_count":4},5909877438554,"https://ap-avatar.wpscdn.com/avatar/5600025385ad2bf12a7?_k=1778553567797529272","arXiv : 2608 . 180 1 1v 1 [cs .CL] 18 Aug 2026  \nThe IOL-AI Challenge:  \nAn Open Challenge towards Advancing Linguistic Reasoning  \nEduardo Sánchez⋆1, Rita Berrada⋆2,3, Dan-Mircea Mirea⋆4, Sara Rajaee5 , Alexander Piperski6 , Ana Meta Dolinar5 , Boris Iomdin7 , Andrey Nikulin8 , Mariya Shmatova9 , Marzieh Fadaee3 , and Julia Kreutzer♦3  \n1University College London & Meta, 2CentraleSupélec & McGill University, 3Cohere Labs, 4Princeton University, 5University of Amsterdam, 6Stockholm University, 7Faculty of Liberal Arts and Sciences, 8Universidade Federal de Goiás, 9Independent  \nAbstract  \nReasoning in LLMs is overwhelmingly studied in domains that provide a model with rules: mathematics and code. Linguistic puzzles invert this: the solver must first discover the system before reasoning within it. We present the IOL-AI Challenge, an open-science competition run on the unseen problems of the International Linguistics Olympiad (IOL) 2026 Individual Contest, evaluated both automatically and, for the first time, by members of the official IOL Jury under the same rubrics applied to human contestants. The challenge drew 731 submissions from 46 teams under a strict compute budget (one T4, 30 mins) . We additionally benchmark 15 unconstrained frontier and open models, with Claude Opus 4.8 earning a jury score equivalent to a gold medal, while both resource-constrained systems we submitted for jury grading scored in the range of the bottom 5% of contestants. Capability was not determined by scale: 14B submissions outperform models twice their size, and gains come from decoding and output-handling rather than model capacity. We also found that automatic metrics rank systems exactly as the jury does, but compress the scale, upscoring weak systems by ∼ 13 points and understating strong ones. Our analysis shows that while frontier models might have prior knowledge about some of the problem languages, it does not significantly help them solve the linguistic reasoning tasks, leaving linguistic reasoning as a strong benchmarking proxy for generalizable reasoning skills.  \n1 Introduction  \nReasoning has become one of the central paradigms for improving the capabilities of large language models (DeepSeek-AI, 2025 ; Marjanovic et al. , 2026 ; Qwen Team, 2026 ; Gemma Team, 2026) . Rather than relying only on larger models, more data, or longer pretraining, recent systems increasingly improve their performance by allocating compute at inference time and generating intermediate reasoning (Snell et al. , 2025 ; Geiping et al. , 2025), exploring alternative paths (Qiu et al. ,  \n⋆ First authors.  \n♦ Principal senior advisor.  \nReleased as a preprint on August 19, 2026 1  \n2026 ; Brown et al. , 2024), verifying solutions, and revising their answers (Hong et al. , 2024 ; Zhang et al. , 2025 ; Liu et al. , 2025) .  \nThe advances have been impressive yet the way we study reasoning has remained remarkably narrow. The dominant tests of reasoning are mathematics (Shen et al. , 2025 ; Chen et al. , 2025a; Zhang et al. , 2026) and coding (Chen et al. , 2025b; He et al. , 2026), which are domains with formal structures, unambiguous answers, and abundant automatically verifiable feedback. However, realworld problems often do not begin with a well-specified system of rules. Before solving the problem, a solver may first need to discover what the relevant structure is. Most reasoning benchmarks abstract away this step: they provide the rules, the formalism, or the problem space, and evaluate how well a system can reason within it. A more fundamental question is whether a system can infer a useful structure from evidence and then reason within the discovered structure . This ability is critical for generalization: Solving new problems in unfamiliar domains requires identifying regularities, forming hypotheses, and testing them against new evidence.  \nNatural language offers a particularly compelling domain in which to explore this question. Human language is an intr","cbCaiadRwfDqgr4J","https://ap.wps.com/l/cbCaiadRwfDqgr4J","pdf",2434642,35,"English","# Abstract\n# 1 Introduction","[{\"question\":\"What makes the IOL-AI Challenge different from typical reasoning benchmarks?\",\"answer\":\"It focuses on linguistic puzzles where the solver must infer the underlying system from evidence before reasoning, rather than being given explicit rules up front.\"},{\"question\":\"How is the competition evaluated?\",\"answer\":\"Solutions are evaluated automatically and, for the first time, also by members of the official IOL Jury using the same rubrics as human contestants.\"},{\"question\":\"What were the main findings about model capability?\",\"answer\":\"Capability was not determined by scale: smaller submissions could outperform larger ones, and automatic metrics agreed with jury rankings while compressing the score range.\"}]","The IOL-AI Challenge - An Open Challenge towards Advancing Linguistic Reasoning | PDF",88]