[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-273468-105":59,"doc-detail-273468-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","counting-documents-is-not-counting-text-unit-bias-in-web-pdf-corpus-statistics","Counting Documents Is Not Counting Text - Unit Bias in Web-PDF Corpus Statistics","","PDF corpora advertise size in tokens but publish rates (coverage, OCR routing, re-fetch recovery, language mix) per document, without decomposing the token totals. Document and token units diverge sharply: on CC-MAIN-2021-31-PDF-UNTRUNCATED, 3.02% of text-bearing documents contain half the tokens (Gini 0.807). PDF truncation is especially damaging: 23.06% of documents but 63.08% of text are affected, with 55–62% of corpus text lost. Reconstructed recovery remains limited and under the 5 MiB cap 30.19% of tokens stay truncated. Recommend reporting statistics in both documents and tokens.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/counting-documents-is-not-counting-text-unit-bias-in-web-pdf-corpus-statistics/273468/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/counting-documents-is-not-counting-text-unit-bias-in-web-pdf-corpus-statistics/273468.png","ImageObject",300,407,{"name":92,"@type":93},"Emma Wilson","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-27","2026-09-15",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",15,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"Why do token-based and document-based statistics differ in web-PDF corpora?","Question",{"text":113,"@type":114},"Because corpora usually report rates per document but headline totals in tokens, without decomposing token sums by the same units. As a result, the distribution of text mass across documents can be highly skewed, making document and token units diverge.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"How much text is lost due to Common Crawl PDF truncation?",{"text":118,"@type":114},"On CC-MAIN-2021-31-PDF-UNTRUNCATED, truncation affects 23.06% of documents but 63.08% of tokens/text. Recovery from reconstructed truncated files captures only part of that content, with roughly 55–62% of the corpus’s text lost overall.",{"name":120,"@type":111,"acceptedAnswer":121},"What change to the truncation cap was introduced in March 2025, and what is its impact?",{"text":122,"@type":114},"The cap increased from 1 MiB to 5 MiB. Even under the 5 MiB limit, 30.19% of tokens would still be truncated, and recovery on affected documents rises only from about 3.3% to 13.2%.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},273468,1789471560,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":29,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":130,"read_time":105,"preview_page_count":4},3848291630094,"https://eur-avatar.wpscdn.com/davatar_085a072bc5b1113ac321206ff7593b45","Preprint. Under review.  \nCounting Documents Is Not Counting Text: Unit Bias in Web-PDF Corpus Statistics  \nLuca Foppiano  \nCommon Crawl Foundation  \n[luca@commoncrawl.org](luca@commoncrawl.org)  \narXiv :2608 . 16390v1 [ cs .CL] 17 Aug 2026  \nAbstract  \nPDF corpora advertise their size in tokens but compute every rate they publish (coverage, OCR routing, re-fetch recovery, language mix) per document, and none decomposes its token total. The two units diverge sharply. On CC-MAIN-2021-31-PDF-UNTRUNCATED (7 .9M web PDFs, 32 .6B tokens), 3 .02% of textbearing documents hold half the tokens (Gini 0.807); documents over 50 pages are 5.00% of the corpus but 53.53% of its text. The PDFs produced by a TEX toolchain are 1.66% of documents and 4 .05% of the text. The clearest casualty is Common Crawl’s truncation cap:  \nit affected 23.06% of documents and 63.08% of the text. Reconstructing the truncated files and extracting both versions, two widely used libraries recover 11.4% and 1.4% of that text;  \nbetween 72% and 97% of affected documents yield nothing; roughly 55–62% of the corpus’s text is lost. Under the 5 MiB cap adopted in March 2025, 30.19% of tokens would still be truncated, and recovery on those documents rises only from 3.3% to 13.2% . We recommend that corpus statistics be reported in both units: documents and tokens.  \n1 Introduction  \nPDFs have become a first-class source of pretraining text (Poznanski et al., 2025 ; Montalvo and Wightman, 2024) . In one current open data pool, PDF-derived text amounts to roughly 4 .3T tokens.1 FinePDFs alone is described as “about 3 trillion tokens across 475 million documents”(Kydlíek et al., 2025) .  \nThose corpora are not naive about tokens: they headline them. What none of them does is compute a rate in tokens, or break a token total down. FinePDFs routes 368.8M of 1.29B files to OCR and  \n1Marin’s datakit source registry: FinePDFs 1,186 .5B English plus 1,353.4B across 18 non-English subsets, and dolma4pdfs 1,804 .0B over 137,132,279 documents. [https:](https:)//[github.com/marin-community/marin](github.com/marin-community/marin)  \nrecovers 53.5% of its truncated files by re-fetching; CCpdf reports per-document success rates (Turski et al., 2023); PDFA gives three totals (2,159,432 documents, 18M pages, 9.7B tokens) but no drop rates in any unit (Montalvo and Wightman, 2024) . No PDF corpus we are aware of decomposes its token total by length or quality.  \nThis would be harmless if documents were interchangeable. They are not. The web-text community already recognises this: Nemotron-CC reports a token-weighted yield,“+57.4% more highquality tokens”(Su et al., 2025), and PDFs have far more extreme length skew than HTML. FinePDFs itself observes that its documents have a “∼5.3k character median (about 2 × other corpora)” with a “95th percentile ∼68k” against “∼ 11–13k elsewhere,” and then reports every rate per document anyway.  \nWe measure what those rates become when weighted by text. Our contributions:  \n1. the first token-weighted characterisation of a web-PDF corpus, with a direct documentversus-token comparison of every headline statistic (§4.1–4.3);  \n2. a concentration result: text mass in web PDFs is far more skewed than document counts suggest (Gini 0.807);  \n3. the first measurement of what Common Crawl’s truncation cap costs in text: 23.06% of documents but 63.08% of tokens, at most  \n11.4% of it recoverable, and 30.19% still lost at the new 5 MiB limit (§4.4–4.6);  \n4. released code.  \n2 Related Work  \nTotals in tokens, rates in documents. FinePDFs (Kydlíek et al., 2025) routes documents between a text path (Docling) and a GPU OCR path with a learned classifier reporting F1 0.71 on the OCR  \nclass, and quantifies every stage in files. CCpdf (Turski et al., 2023) tabulates “number of documents per processing step and language” and reports no token count for its own corpus. PDFA (Montalvo and Wightman, 2024), derived from the same corpus we study, filters per document, dis","cbCaiavHFXctmjeA","https://ap.wps.com/l/cbCaiavHFXctmjeA","pdf",133200,"English","# Abstract\n# Introduction\n# Related Work\n# Data and Method","[{\"question\":\"Why do token-based and document-based statistics differ in web-PDF corpora?\",\"answer\":\"Because corpora usually report rates per document but headline totals in tokens, without decomposing token sums by the same units. As a result, the distribution of text mass across documents can be highly skewed, making document and token units diverge.\"},{\"question\":\"How much text is lost due to Common Crawl PDF truncation?\",\"answer\":\"On CC-MAIN-2021-31-PDF-UNTRUNCATED, truncation affects 23.06% of documents but 63.08% of tokens/text. Recovery from reconstructed truncated files captures only part of that content, with roughly 55–62% of the corpus’s text lost overall.\"},{\"question\":\"What change to the truncation cap was introduced in March 2025, and what is its impact?\",\"answer\":\"The cap increased from 1 MiB to 5 MiB. Even under the 5 MiB limit, 30.19% of tokens would still be truncated, and recovery on affected documents rises only from about 3.3% to 13.2%.\"}]","Counting Documents Is Not Counting Text - Unit Bias in Web-PDF Corpus Statistics | PDF"]