[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-657560-105":59,"doc-detail-657560-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","structeval-benchmarking-llms-capabilities-to-generate-structural-outputs","StructEval - Benchmarking LLMs' Capabilities to Generate Structural Outputs","","As Large Language Models become integral to software development workflows, their ability to generate structured outputs is critically important. StructEval introduces a benchmark evaluating non-renderable formats (JSON, YAML, CSV) and renderable formats (HTML, React, SVG). It compares two paradigms: generation from natural language prompts and conversion between structured formats. The benchmark covers 18 formats and 44 task types with new metrics for format adherence and structural correctness, revealing sizable gaps across models.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/structeval-benchmarking-llms-capabilities-to-generate-structural-outputs/657560/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/structeval-benchmarking-llms-capabilities-to-generate-structural-outputs/657560.png","ImageObject",300,407,{"name":92,"@type":93},"eBook King","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-30","2026-10-26",true,{"@type":102,"interactionType":103,"userInteractionCount":14},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What does StructEval benchmark?","Question",{"text":112,"@type":113},"StructEval benchmarks LLM capabilities for producing structured outputs across both non-renderable formats (e.g., JSON, YAML, CSV) and renderable formats (e.g., HTML, React, SVG).","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does StructEval evaluate structured output?",{"text":117,"@type":113},"It uses two task paradigms: generation tasks that convert natural language prompts into structured outputs, and conversion tasks that translate between structured formats. It also introduces metrics for format adherence and structural correctness.",{"name":119,"@type":110,"acceptedAnswer":120},"Which tasks are harder in StructEval’s results?",{"text":121,"@type":113},"Generation tasks are more challenging than conversion tasks, and producing correct visual content is harder than generating text-only structures.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},657560,1792982824,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":14,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":129,"read_time":26},962088006270,"https://ap-avatar.wpscdn.com/davatar_085a072bc5b1113ac321206ff7593b45","arXiv :2505 .20 139v 3 [ cs . SE] 2 Apr 2026  \nStructEval: Benchmarking LLMs’ Capabilities to Generate Structural Outputs  \nJialin Yang∗ † 1 Dongfu Jiang∗ † 1 Lipeng He 1 Sherman Siu 1 Yuxuan Zhang7 Disen Liao 1 Zhuofeng Li4 Huaye Zeng 1 Yiming Jia 1 Haozhe Wang3 Benjamin Schneider 1 Chi Ruan5 Wentao Ma 1 Zhiheng Lyu 1 Yifei Wang 1 Yi Lu2 Quy Duc Do 1 Ziyan Jiang 1 Ping Nie 1 Wenhu Chen† 6  \n1 University of Waterloo, 2 University of Toronto, 3 HKUST, 4 Shanghai University, 5 Independent Contributor, 6 Vector Institute 7 University of British Columbia ∗ Equal Contribution †{j586yang, dongfu.jiang, wenhuchen, [ping.nie}@uwaterloo.ca](ping.nie}@uwaterloo.ca)  \nReviewed on OpenReview: [https://openreview.net/forum?id=buDwV7LUA7](https://openreview.net/forum?id=buDwV7LUA7)  \nAbstract  \nAs Large Language Models (LLMs) become integral to software development workflows, their ability to generate structured outputs has become critically important. We introduce StructEval, a comprehensive benchmark for evaluating LLMs’ capabilities in producing both non-renderable (JSON, YAML, CSV) and renderable (HTML, React, SVG) structured formats. Unlike prior benchmarks, StructEval systematically evaluates structural fidelity across diverse formats through two paradigms: 1) generation tasks, producing structured output from natural language prompts, and 2) conversion tasks, translating between structured formats. Our benchmark encompasses 18 formats and 44 types of task, with novel metrics for format adherence and structural correctness. Results reveal significant performance gaps—even state-of-the-art models like o1-mini achieve only 75.58 average score, with open-source alternatives lagging approximately 10 points behind. We find generation tasks more challenging than conversion tasks, and producing correct visual content more difficult than generating text-only structures. Please see our project page for more detail [https://tiger-ai-lab.github.io/StructEval/](https://tiger-ai-lab.github.io/StructEval/) .  \n1 Introduction  \nIn recent years, there has been a significant surge in the capabilities of large language models (LLMs) in generating human-like text and performing a wide range of natural language processing tasks. State-ofthe-art models like GPT-4o (Hurst et al., 2024), OpenAI o1/o3 (Contributors et al., 2024), and Google’s Gemini (Team et al., 2023) have achieved superior performance in knowledge QA (Hendrycks et al., 2020; Wang et al., 2024), instruction-following (Chiang et al., 2024; Zhou et al., 2023), and code generation (Zhuo et al., 2024; Jain et al., 2024) .  \nDespite recent advances, many real-world applications require not only fluency in the content of the output but also precise control over its structure. This includes tasks where the expected output must follow specific formats such as JSON, XML, LaTeX, HTML, or code in frameworks like React or Vue. Additionally, in these tasks, we also want the code to render a page that correctly places elements according to the requirements. These types of structured output are essential in domains like software development, data pipelines, user interface generation, and scientific publishing, where incorrect formatting can lead to disrupted pipelines or non-functional outputs.  \nHowever, most existing benchmarks focus on the semantic quality (Wang et al., 2024) or reasoning ability of LLMs (Hendrycks et al., 2021; He et al., 2024), with limited emphasis on their ability to produce  \nFigure 1: StructEval evaluates the LLM’s capability to generate structured outputs, including text-only tasks like JSON, TOML, etc, and visual rendering tasks like HTML, React, Latex, etc.  \nFigure 2: The overall designed annotation pipeline of StructEval dataset  \nformat-conforming structured outputs. Some recently proposed benchmarks aim to evaluate the quality of structured outputs tend to target specific modalities, such as code generation (Zhuo et al., 2024) or text-only structures (Gu et al., 2024; Tang","cbCaiqIeLFCTdqaR","https://ap.wps.com/l/cbCaiqIeLFCTdqaR","pdf",1850108,24,"English","# Introduction\n## Structured output needs precise control\n## Limitations of existing benchmarks\n## Challenges in evaluation\n## StructEval overview","[{\"question\":\"What does StructEval benchmark?\",\"answer\":\"StructEval benchmarks LLM capabilities for producing structured outputs across both non-renderable formats (e.g., JSON, YAML, CSV) and renderable formats (e.g., HTML, React, SVG).\"},{\"question\":\"How does StructEval evaluate structured output?\",\"answer\":\"It uses two task paradigms: generation tasks that convert natural language prompts into structured outputs, and conversion tasks that translate between structured formats. It also introduces metrics for format adherence and structural correctness.\"},{\"question\":\"Which tasks are harder in StructEval’s results?\",\"answer\":\"Generation tasks are more challenging than conversion tasks, and producing correct visual content is harder than generating text-only structures.\"}]","StructEval - Benchmarking LLMs' Capabilities to Generate Structural Outputs | PDF"]