[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-267746-105":59,"doc-detail-267746-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","prior-based-noisy-text-data-filtering-fast-and-strong-alternative-for-perplexity","PRIOR-BASED NOISY TEXT DATA FILTERING - FAST AND STRONG ALTERNATIVE FOR PERPLEXITY","","Large language models rely on massive web corpora, making data selection critical for both effective and efficient pretraining. Prior work using perplexity (PPL) filtering can achieve strong results but is slow and unreliable on noisy or out-of-distribution samples. This paper proposes a prior-based filtering approach that estimates token priors via corpus term-frequency statistics, avoiding model inference. It selects documents using mean and standard deviation of token priors, matching PPL as a fast proxy.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/prior-based-noisy-text-data-filtering-fast-and-strong-alternative-for-perplexity/267746/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/prior-based-noisy-text-data-filtering-fast-and-strong-alternative-for-perplexity/267746.png","ImageObject",300,407,{"name":92,"@type":93},"Emma Wilson","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-30","2026-09-14",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",17,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"Why is data filtering necessary for pretrained LLMs?","Question",{"text":113,"@type":114},"Web corpora are massive and noisy. Efficient selection reduces computational waste and prevents low-quality data from harming training.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"What are the main drawbacks of PPL-based filtering?",{"text":118,"@type":114},"It is prohibitively time-consuming because it requires reference-model inference over the full corpus, and it can be unreliable on noisy or out-of-distribution text.",{"name":120,"@type":111,"acceptedAnswer":121},"How does the proposed prior-based filtering work?",{"text":122,"@type":114},"It estimates token priors using corpus-level term-frequency statistics, then filters documents based on the mean and standard deviation of these token priors without requiring model inference.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},267746,1789424621,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":130,"read_time":145,"preview_page_count":4},3848291630094,"https://eur-avatar.wpscdn.com/davatar_085a072bc5b1113ac321206ff7593b45","PRIOR-BASED NOISY TEXT DATA FILTERING:  \nFAST AND STRONG ALTERNATIVE FOR PERPLEXITY  \nYeongbin Seo Gayoung Kim Jaehyung Kim † Jinyoung Yeo † Department of Artificial Intelligence  \nYonsei University  \n{suhcrates,kykim,jaehyungk, [jinyeo}@yonsei.ac.kr](jinyeo}@yonsei.ac.kr)  \nABSTRACT  \nAs large language models (LLMs) are pretrained on massive web corpora, careful selection of data becomes essential to ensure effective and efficient learning. While perplexity (PPL)-based filtering has demonstrated strong performance, it suffers from drawbacks: substantial time costs and inherent unreliability of the model when handling noisy or out-of-distribution samples. In this work, we propose a simple yet powerful alternative: a prior-based data filtering method that estimates token priors using corpus-level term frequency statistics, inspired by linguistic insights on word roles and lexical density. Our approach filters documents based on the mean and standard deviation of token priors, serving as a fast proxy to PPL while requiring no model inference. Despite its simplicity, the prior-based filter achieves the highest average performance across 20 downstream benchmarks, while reducing time cost by over 1000× compared to PPL-based filtering. We further demonstrate its applicability to symbolic languages such as code and math, and its dynamic adaptability to multilingual corpora without supervision. The code is available online ([https://github.com/ybseo-ac/prior_filter](https://github.com/ybseo-ac/prior_filter)) .  \n1 INTRODUCTION  \nLarge Language Models (LLMs) have achieved impressive performance by training on massive datasets, with web text serving as a primary data source. As web content continues to grow indefinitely, it offers unlimited data for pretraining. However, two major challenges necessitate careful filtering steps: (1) Web data is so large that we need to choose efficiently to save computational resources, and (2) It contains a lot of noise, which can harm the model if not properly filtered.  \nTo address this need, various data selection methods have been proposed. Early approaches relied on heuristic rules (Raffel et al., 2023 ; Bane et al., 2022), but more recent trends have shifted toward model-based techniques (Xie et al., 2023b ; Marion et al., 2023) . These methods typically involve training a reference model on a target dataset and using it to identify desirable data. The model may perform binary classification (Xie et al., 2023a) or compute similarity with the reference dataset (Xie et al., 2023b) . Among these, using the perplexity (PPL) score from a reference model as a criterion of filtering is currently known to offer the best performance while maintaining a relatively simple implementation (Ankner et al., 2024) . We provide a more detailed review of related work in §A.  \nHowever, PPL-based approaches come with the following inherent limitations. (1) Time cost: These methods require training a reference model, followed by inference of PPL over the whole corpus. Given that web-scale data can easily exceed trillions of documents and continues to grow, performing inference over the entire corpus becomes prohibitively expensive. (2) Reliability: LLMs often fail to accurately assess samples from distributions that is not seen while training, such as noisy data. As a result, generative perplexity may sometimes assign high scores to noisy or low-quality text (Holtzmanet al., 2020 ; Welleck et al., 2019) . This issue might become more pronounced when using smaller models to reduce inference costs, further undermining reliability.  \n†Co-corresponding authors  \nTo address this limitation of the PPL-based approach, we introduce a prior-based data filtering method grounded in linguistic insights. Instead of computing the full conditional probability of each token in the data p(xi |x\u003Ci) ∝ p(x\u003Ci |xi)p (xi) (xi is token of a data d), this method focuses solely on estimating the prior term p(xi) with statistical metric such as ","cbCairKmjYfz424d","https://ap.wps.com/l/cbCairKmjYfz424d","pdf",7074369,32,"English","# Abstract\n# Introduction\n## Challenges with web data and the need for filtering\n## Limitations of perplexity (PPL) based filtering\n## Prior-based data filtering method\n## Linguistic motivation and core idea\n# Experiments and results","[{\"question\":\"Why is data filtering necessary for pretrained LLMs?\",\"answer\":\"Web corpora are massive and noisy. Efficient selection reduces computational waste and prevents low-quality data from harming training.\"},{\"question\":\"What are the main drawbacks of PPL-based filtering?\",\"answer\":\"It is prohibitively time-consuming because it requires reference-model inference over the full corpus, and it can be unreliable on noisy or out-of-distribution text.\"},{\"question\":\"How does the proposed prior-based filtering work?\",\"answer\":\"It estimates token priors using corpus-level term-frequency statistics, then filters documents based on the mean and standard deviation of these token priors without requiring model inference.\"}]","PRIOR-BASED NOISY TEXT DATA FILTERING - FAST AND STRONG ALTERNATIVE FOR PERPLEXITY | PDF",81]