[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-273693-105":59,"doc-detail-273693-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","formaltcs-benchmarking-end-to-end-frontier-formal-theoretical-computer-science-research-of-large-language-models-abstract","FORMALTCS - Benchmarking End-to-End Frontier Formal Theoretical Computer Science Research of Large Language Models - Abstract","","Large language models (LLMs) have demonstrated increasing ability to support theoretical computer science (TCS) research, yet existing benchmarks do not reflect realistic research workflows. FORMALTCS provides an expert-validated benchmark that evaluates LLMs on frontier, end-to-end TCS tasks using 143 instances from top conferences (STOC, FOCS, SODA, COLT). Each instance includes paper-specific definitions, assumptions, proof dependencies, and expert-verified Lean formalizations, with theorem statements and proof processes annotated in both natural and formal languages.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/formaltcs-benchmarking-end-to-end-frontier-formal-theoretical-computer-science-research-of-large-language-models-abstract/273693/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/formaltcs-benchmarking-end-to-end-frontier-formal-theoretical-computer-science-research-of-large-language-models-abstract/273693.png","ImageObject",300,407,{"name":92,"@type":93},"Sage","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-11","2026-09-15",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",17,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What does FORMALTCS benchmark evaluate for LLMs in TCS research?","Question",{"text":113,"@type":114},"FORMALTCS evaluates LLMs on frontier, end-to-end TCS research by assessing multiple stages needed to transform natural-language core claims into rigorous Lean proofs.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"How is FORMALTCS constructed and validated?",{"text":118,"@type":114},"FORMALTCS contains 143 instances from papers accepted to STOC, FOCS, SODA, and COLT in 2025–2026. It preserves paper-specific definitions, assumptions, and proof dependencies, and includes expert-verified Lean formalizations and proofs.",{"name":120,"@type":111,"acceptedAnswer":121},"What bottleneck do evaluations show for current LLMs?",{"text":122,"@type":114},"Autoformalization is identified as the sharpest bottleneck: the best model achieves 11.5 on translating natural-language claims into formal theorem statements, compared with 28.6 Pass@8 when proving human-provided formal statements.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},273693,1789472514,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":130,"read_time":145,"preview_page_count":4},687197207057,"https://ap-avatar.wpscdn.com/davatar_29158cc5080c5b710cf443261637dec0","arXiv : 2608 .20 153v2 [cs .CL] 1 Sep 2026  \nFORMALTCS: BENCHMARKING END-TO-END FRONTIER FORMAL THEORETICAL COMPUTER SCIENCE RESEARCH OF LARGE LANGUAGE MODELS  \nDingzirui Wang Xuanliang Zhang Keyan Xu Qingfu Zhu Wanxiang Che  \nHarbin Institute of Technology  \n{dzrwang,xuanliangzhang,kyxu,qfzhu,[car](car}@ir.hit.edu.cn)[}](car}@ir.hit.edu.cn)[@ir.hit.edu.cn](car}@ir.hit.edu.cn)  \nABSTRACT  \nLarge language models (LLMs) have shown growing capabilities for automated theoretical computer science (TCS) research, while existing benchmarks remain far from realistic research settings. We introduce FORMALTCS, an expertvalidated benchmark for evaluating LLMs on frontier, end-to-end TCS research.  \nFORMALTCS contains 143 instances drawn from papers accepted to STOC, FOCS, SODA, and COLT in 2025-2026, preserving paper-specific definitions, assumptions, and proof dependencies, with expert-verified Lean formalizations and proofs. For each instance, we annotate the theorem statements and proof processes in both natural and formal languages to evaluate the main bottlenecks of existing LLMs in TCS. Evaluations of leading LLMs reveal that current models remain far from reliably completing the full TCS research pipeline. In particular, we reveal that autoformalization is the sharpest bottleneck of TCS auto research, where the best model achieves only 11.5 for translating natural-language claims into formal theorem statements, compared with 28.6 Pass@8 when proving humanprovided formal statements. Building on FORMALTCS, we further develop an automated TCS research framework that generates, formalizes, filters, and proves new claims. Of 64 generated claims, only 6 ultimately pass expert evaluation and proof verification, indicating that beyond formalization, limited research taste remains another major barrier to autonomous TCS research 1.  \n1 INTRODUCTION  \nTheoretical computer science (TCS) studies the fundamental principles of computation through mathematical methods, including models of computation, algorithms, computational complexity, and the limits of computability (van Leeuwen, 1990; Wigderson, 2019) . TCS is an important research topic, as it provides the theoretical foundation for understanding which problems can be computed and how efficiently they can be solved. Given the fundamental importance of TCS and the increasingly strong autonomous research capabilities demonstrated by large language models (LLMs) (Chen et al., 2025), a growing body of work has begun to investigate how well LLMs can perform TCS-related research tasks. For example, LCS-Bench (Feng et al., 2026b) constructs a benchmark from TCS knowledge extracted from textbooks, while TCS-Bench (Cohen-Addad et al., 2026) evaluates LLMs to generate proofs of TCS theorems from natural-language statements.  \nHowever, existing TCS benchmarks still exhibit a substantial gap from real-world TCS research due to the following limitations: (i) Incomplete research process. Existing TCS benchmarks primarily evaluate isolated capabilities, such as autoformalization or proof generation. They do not provide an end-to-end evaluation of whether LLMs can conduct TCS research from scratch, making it difficult to identify where current models fail throughout the research process. (ii) Outdated TCS content. Existing benchmarks are largely constructed from textbook material or theorems already available in libraries such as Mathlib (Moura & Ullrich, 2021) . As a result, they provide limited insight into an LLM’s ability to reason about frontier TCS research and may also suffer from contamination when benchmark theorems or closely related material have appeared in the model’s training data.  \n1Our code and benchmark are released in [https://github.com/zirui-HIT/FormalTCS](https://github.com/zirui-HIT/FormalTCS).  \n\n| Finding | Evidence |\n| --- | --- |\n| Current LLMs remain far from TCS research end-to-end | §4.2 |\n| Generating formal definitions and theorem statements is the sharpest bottleneck of au","cbCaidldAaBmJaRQ","https://ap.wps.com/l/cbCaidldAaBmJaRQ","pdf",570569,26,"English","# Abstract\n## End-to-end TCS research benchmarking\n## Benchmark construction and expert validation\n## Reported limitations and evaluation outcomes\n## Introduction and motivation","[{\"question\":\"What does FORMALTCS benchmark evaluate for LLMs in TCS research?\",\"answer\":\"FORMALTCS evaluates LLMs on frontier, end-to-end TCS research by assessing multiple stages needed to transform natural-language core claims into rigorous Lean proofs.\"},{\"question\":\"How is FORMALTCS constructed and validated?\",\"answer\":\"FORMALTCS contains 143 instances from papers accepted to STOC, FOCS, SODA, and COLT in 2025–2026. It preserves paper-specific definitions, assumptions, and proof dependencies, and includes expert-verified Lean formalizations and proofs.\"},{\"question\":\"What bottleneck do evaluations show for current LLMs?\",\"answer\":\"Autoformalization is identified as the sharpest bottleneck: the best model achieves 11.5 on translating natural-language claims into formal theorem statements, compared with 28.6 Pass@8 when proving human-provided formal statements.\"}]","FORMALTCS - Benchmarking End-to-End Frontier Formal Theoretical Computer Science Research of Large Language Models - Abstract | PDF",66]