[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-327500-105":59,"doc-detail-327500-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","testevo-bench-an-executable-and-live-benchmark-for-test-and-code-co-evolution","TESTEVO-BENCH - An Executable and Live Benchmark for Test and Code Co-Evolution","","Software tests and code evolve together, since a code change should trigger new or updated tests that record the revised behavior. Existing benchmarks often separate test generation or updating from code-change context and rely on static metadata, making it hard to judge whether an agent captures the intended propagation. TESTEVO-BENCH provides executable, live, commit-anchored co-evolution tasks with environment configuration and execution-grounded metrics such as pass rate, coverage, and mutation score.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/testevo-bench-an-executable-and-live-benchmark-for-test-and-code-co-evolution/327500/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/testevo-bench-an-executable-and-live-benchmark-for-test-and-code-co-evolution/327500.png","ImageObject",300,407,{"name":92,"@type":93},"Hazel","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-12","2026-09-21",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",15,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"Why do software tests and code need to co-evolve in benchmarks?","Question",{"text":113,"@type":114},"Because a code change should be followed by new or updated tests that record the new software behavior. Benchmarks that isolate tests from code-change context cannot reliably evaluate an agent’s understanding of the required propagation.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"What tracks does TESTEVO-BENCH provide?",{"text":118,"@type":114},"It provides two tracks: test generation, where agents write new tests capturing behavior from a code change, and test update, where agents adapt failing existing tests to match the changed behavior.",{"name":120,"@type":111,"acceptedAnswer":121},"How is TESTEVO-BENCH evaluated and why is it executable and live?",{"text":122,"@type":114},"Tasks are anchored to real commit history and packaged with environment configuration to enable execution-grounded metrics like success rate, pass rate, coverage, and mutation score. It is live because new tasks are periodically mined with timestamps and can be filtered to avoid training cutoff leakage.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},327500,1790403951,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":145,"read_time":146,"preview_page_count":4},137441390410,"https://ap-avatar.wpscdn.com/avatar/2000252f4ab5702993?_k=1776741390130283984","arXiv :2607 .02469v 1 [ cs . SE] 2 Jul 2026  \nTESTEVO-BENCH: An Executable and Live Benchmark for Test and Code Co-Evolution  \nJiale Amber Wang 1 Kaiyuan Wang2 Pengyu Nie 1  \n1University of Waterloo 2 Google  \n[jiale.wang@uwaterloo.ca](jiale.wang@uwaterloo.ca) , [kaiyuanw@google.com](kaiyuanw@google.com) , [pynie@uwaterloo.ca](pynie@uwaterloo.ca)  \nAbstract  \nSoftware tests and code evolve together: a code change should be followed by new or updated tests that record the new software behavior. Yet existing test generation and update benchmarks often isolate the test from the code change, and rely on static metadata that does not verify whether a test is executable or semantically tied to the code change. This makes it difficult to evaluate whether a test automation agent understands how a code change should propagate into the test suite.  \nWe introduce TESTEVO-BENCH, a benchmark of test and code co-evolution tasks mined from software repositories, with two tracks: in test generation, the agentshall write new tests to capture the new software behavior; in test update, the agentshall adapt failing existing tests to the changed software behavior. Each task is anchored to a real commit history and packaged with environment configuration to support execution-grounded metrics such as pass rate, coverage, and mutation score.  \nTESTEVO-BENCH is also a live benchmark: each task records the timestamp of the test and code changes, and new tasks are periodically mined by our automated pipeline, so evaluation can be restricted to tasks postdating a model’s training cutoff to reduce data leakage risk. The current snapshot contains 746 test generation and  \n509 test update tasks, curated from 59,950 candidate co-evolution records across  \n152 open-source Java projects. We experiment with four state-of-the-art agents that combine strong harnesses (Claude Code, Gemini CLI, and SWE-Agent) with strong foundation models (Claude Opus 4.7 and Gemini 3.1 Pro) . Results show that they achieve up to 77.5% success rate on test generation and 74.6% on test update. However, success rate is materially lower on the most recent benchmark tasks and drops significantly under limited per-task cost. 1  \n1 Introduction  \nRegression tests encode expectations about software behavior. When code changes, developers often update existing tests or write additional tests in the same or subsequent revision to record the new software behavior. Prior studies of test and code co-evolution show that this synchronization is central to software development and maintenance (Zaidman et al., 2011; Lubsen et al., 2009; Marsavina et al., 2014; Levin and Yehudai, 2017; Le Dilavrec et al., 2021) .2 A test edit is therefore not merely another code change, but evidence of how developers believed the code edit should be exercised.  \nCurrent benchmarks and evaluation frameworks do not capture this signal well. Test generation benchmarks usually ask a system to synthesize tests for a fixed code snapshot rather than for a code change (Fraser and Arcuri, 2011; Pacheco et al., 2007; Lops et al., 2025) . Prior work on test update addresses one important part of this problem: adapting existing tests after code changes (Hu et al.,  \n1 TESTEVO-BENCH leaderboard and data explorer are hosted at [https://www.testevo-bench.com](https://www.testevo-bench.com).  \n2We use the general term “code” to denote the non-testing part of the codebase, which some studies call “production code.”  \nPreprint.  \n2023; Saboor Yaraghi et al., 2025; Liu et al., 2024; Zhang et al., 2025); however, it does not cover additional tests added to the test suite, which are another common way developers record changed behavior. Across both settings, many datasets rely on static dependency analysis and diff signals as labels, without preserving enough execution context to rebuild the project, run the affected tests, or verify that an updated or generated test actually reflects the code change.  \nRecent large language models and","cbCaigxIdmIbHzGw","https://ap.wps.com/l/cbCaigxIdmIbHzGw","pdf",1395580,22,"English","# Abstract\n# Introduction","[{\"question\":\"Why do software tests and code need to co-evolve in benchmarks?\",\"answer\":\"Because a code change should be followed by new or updated tests that record the new software behavior. Benchmarks that isolate tests from code-change context cannot reliably evaluate an agent’s understanding of the required propagation.\"},{\"question\":\"What tracks does TESTEVO-BENCH provide?\",\"answer\":\"It provides two tracks: test generation, where agents write new tests capturing behavior from a code change, and test update, where agents adapt failing existing tests to match the changed behavior.\"},{\"question\":\"How is TESTEVO-BENCH evaluated and why is it executable and live?\",\"answer\":\"Tasks are anchored to real commit history and packaged with environment configuration to enable execution-grounded metrics like success rate, pass rate, coverage, and mutation score. It is live because new tasks are periodically mined with timestamps and can be filtered to avoid training cutoff leakage.\"}]","TESTEVO-BENCH - An Executable and Live Benchmark for Test and Code Co-Evolution | PDF",1789977348,55]