[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-273398-105":59,"doc-detail-273398-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","detecting-functional-memorization-in-code-language-models","Detecting Functional Memorization in Code Language Models","","Large language models increasingly generate code at scale, yet existing leakage audits rely on textual overlap and miss logic-level exposure. This work studies functional memorization, where models reproduce training data logic despite substantial syntactic and structural differences. Using AI coding agents to generate diverse test inputs, the authors formalize a counterfactual evaluation comparing target and reference models under functional equivalence. Across four open-source models and five languages, 0.3–3.4% of filtered functions show counterfactual functional memorization.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/detecting-functional-memorization-in-code-language-models/273398/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/detecting-functional-memorization-in-code-language-models/273398.png","ImageObject",300,407,{"name":92,"@type":93},"Anda","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-18","2026-09-15",true,{"@type":102,"interactionType":103,"userInteractionCount":39},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"Why do textual auditing methods fail to detect some code memorization?","Question",{"text":112,"@type":113},"Because code may be functionally equivalent while being textually dissimilar through renamed variables, altered control flow, or restructured logic. A model can internalize logic and reproduce it in a different form that low textual overlap metrics miss.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does the paper evaluate functional memorization?",{"text":117,"@type":113},"It uses AI coding agents to generate diverse test inputs for functions from training data, then checks whether model-generated continuations match the same outputs. The evaluation is formalized with a counterfactual framework comparing target versus reference models and requiring functional equivalence for the target.",{"name":119,"@type":110,"acceptedAnswer":120},"What empirical findings are reported across open-source models and programming languages?",{"text":121,"@type":113},"The study instantiates the framework across four open-source models and filters for functions with meaningful logic in five programming languages. It finds 0.3–3.4% of filtered functions are counterfactually functionally memorized, and that functional memorization correlates with semantic duplication in the training corpus.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},273398,1789471251,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":39,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":129,"read_time":144,"preview_page_count":4},962075006959,"https://ap-avatar.wpscdn.com/avatar/e0002397efbe92a78e?_k=1776741047341049297","arXiv : 2606 . 12764v2 [cs .LG] 20 Aug 2026  \nDetecting Functional Memorization in Code Language Models  \nMatthieu Meeus 1 ,2 , Anil Ramakrishna 1 , Shengyuan Hu 1 , Matthew Grange 1 , Zheng Xu 1 , Luca Melis 1 1 Meta, 2 Imperial College London  \nLarge language models (LLMs) are increasingly used to generate code at scale. Meanwhile, prior work has investigated whether training data may be recoverable from model outputs, by auditing the textual overlap between training examples and model generations. Code, however, can preserve the same logic while differing substantially in syntax and structure. We here study functional memorization: the leakage of training data logic from LLM generations in ways that textual audits fail to detect. We leverage AI coding agents to generate diverse test inputs for training data functionality and evaluate whether model-generated continuations produce the same outputs. We formalize this through a counterfactual framework, comparing target models (exposed to specific code) against reference models (not exposed) and requiring functional equivalence only for the target. We instantiate this framework across 4 open-source models and explicitly filter for functions with meaningful logic in 5 programming languages. We find that 0 .3–3.4% of filtered functions are counterfactually functionally memorized, i.e., reproduced with equivalent behavior in restructured code that textual metrics fail to detect. We further show that LLM-based judges offer a scalable proxy for execution-based testing, achieving a true positive rate of 68% at 1% false positive rate, and find that functional memorization is associated with semantic duplication in the training corpus.  \nCorrespondence: Luca Melis at [lucamelis@meta.com](lucamelis@meta.com)   \n1 Introduction  \nLarge language models (LLMs) are increasingly used to generate code, both as standalone models (Roziereet al. , 2023 ; Feng et al. , 2020 ; Olmo et al. , 2025 ; Li et al. , 2022) and as core components of coding agents (Zeff, 2026 ; Thompson, 2026) . These capabilities are acquired by training on large corpora of source code, often scraped from public repositories with permissive licenses (Allal et al. , 2025 ; Li et al. , 2022 ; Roziere et al. , 2023) . However, filtering out non-permissively licensed code is difficult (Katzy et al. , 2024), while training data is also increasingly sourced from user coding sessions (GitHub, 2026 ; Loizos, 2025) .  \nAt the same time, prior work has studied the extent to which training data might be recoverable from model outputs, as studied extensively for natural language (Carlini et al. , 2021 ; Nasr et al. , 2025 ; Ippolito et al. , 2023) . Similar techniques have also been applied to code, measuring near-verbatim leakage of source code (Al-Kaswan et al. , 2024 ; Salerno et al. , 2025) and secrets such as API keys (Nie et al. , 2025 ; Huang et al. , 2024) .  \nHowever, existing work on code leakage focuses almost exclusively on textual similarity: does the model’s output match the training data token-for-token, or nearly so? This framing misses a critical dimension when models are trained on code containing proprietary logic, e.g. , algorithms for recommendation systems, trading, or content moderation, that owners would reasonably want to protect. Indeed, code can be functionally equivalent while being textually dissimilar: variables can be renamed, comments added or removed, control flow restructured, or entirely different algorithms used to achieve the same functionality—as has been widely studied in code clone detection (Roy and Cordy, 2007 ; Ren et al. , 2020 ; Song et al. , 2024) . Hence, if a model internalizes logic from its training data and reproduces it in a different form, memorization auditing based on textual similarity will fail to detect it.  \nContributions. We here formalize functional memorization: given a training data function x and a generation x⋆T from a target model prompted with the function signature, we","cbCaifXSaFmMcCta","https://ap.wps.com/l/cbCaifXSaFmMcCta","pdf",2031350,52,"English","# Introduction\n## Functional memorization vs textual leakage\n## Contributions and evaluation approach\n## Execution-based testing example","[{\"question\":\"Why do textual auditing methods fail to detect some code memorization?\",\"answer\":\"Because code may be functionally equivalent while being textually dissimilar through renamed variables, altered control flow, or restructured logic. A model can internalize logic and reproduce it in a different form that low textual overlap metrics miss.\"},{\"question\":\"How does the paper evaluate functional memorization?\",\"answer\":\"It uses AI coding agents to generate diverse test inputs for functions from training data, then checks whether model-generated continuations match the same outputs. The evaluation is formalized with a counterfactual framework comparing target versus reference models and requiring functional equivalence for the target.\"},{\"question\":\"What empirical findings are reported across open-source models and programming languages?\",\"answer\":\"The study instantiates the framework across four open-source models and filters for functions with meaningful logic in five programming languages. It finds 0.3–3.4% of filtered functions are counterfactually functionally memorized, and that functional memorization correlates with semantic duplication in the training corpus.\"}]","Detecting Functional Memorization in Code Language Models | PDF",131]