[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-273621-105":59,"doc-detail-273621-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","institutional-booksenriched-text-a-customizable-multilingual-open-source-pipeline-for-denoising-deduplicating-and-annotating-ocr-text-at-scale","Institutional Books—Enriched Text - A customizable multilingual open-source pipeline for denoising, deduplicating, and annotating OCR text at scale","","Released in 2025, Institutional Books: Harvard Library (IB-HL) is a large multilingual collection of digitized volumes, whose scale creates a conflict between web-oriented preprocessing and careful information stewardship. Existing pipelines can over-filter, deduplicate aggressively, language-restrict, and drop meaningful metadata, while researchers need consistent reuse across duplicate work. This report presents Enriched Text, which normalizes OCR text while preserving metadata through HTML-like annotations, enabling endmatter handling, per-paragraph language detection, duplicate cluster identification, and per-paragraph bits-per-byte scoring tailored to user needs. The release includes IB-HL-ET and the pipeline that produced it.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/institutional-booksenriched-text-a-customizable-multilingual-open-source-pipeline-for-denoising-deduplicating-and-annotating-ocr-text-at-scale/273621/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/institutional-booksenriched-text-a-customizable-multilingual-open-source-pipeline-for-denoising-deduplicating-and-annotating-ocr-text-at-scale/273621.png","ImageObject",300,407,{"name":92,"@type":93},"Theodore","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-17","2026-09-15",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",12,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What problem does the Enriched Text approach address in IB-HL preprocessing?","Question",{"text":113,"@type":114},"It addresses the mismatch between common web-text preprocessing and careful information stewardship, where pipelines may filter, deduplicate, restrict languages, or discard valuable metadata. Enriched Text preserves metadata via annotations while still normalizing and preparing the text at scale.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"How does Enriched Text preserve information while normalizing OCR text?",{"text":118,"@type":114},"Instead of outputting a single complete token stream, it normalizes text while preserving metadata through HTML-like layered annotations. These annotations support user-driven tailoring rather than a single global editorial decision.",{"name":120,"@type":111,"acceptedAnswer":121},"What processing steps are included in the pipeline?",{"text":122,"@type":114},"The pipeline includes preprocessing, Unicode normalization, duplicate page removal, endmatter separation, dehyphenation, removal of running headers/footers and page numbers, sentence segmentation, topical chunking, duplicate identification, bits-per-byte computation, and final annotation/formatting.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},273621,1789472299,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":130,"read_time":145,"preview_page_count":4},7971461740886,"https://ap-avatar.wpscdn.com/davatar_3d24733baf745e90a7e4bdd5f77d97b2","arXiv : 2608 . 19026v 1 [cs .CL] 19 Aug 2026  \nInstitutional Books—Enriched Text  \nA customizable multilingual open-source pipeline for denoising, deduplicating, and annotating OCR text at scale  \nDavid Lowry-Duda1 , Matteo Cargnelutti1 , Catherine Brobston1 , Salwa Ismail2 , Greg Leppert†1, Amanda Watson3 , and Jonathan Zittrain4  \n1 Institutional Data Initiative, Harvard Law School Library  \n2 Harvard Library  \n3 Harvard Law School Library  \n4 Harvard Law School, Harvard School of Engineering and Applied Sciences, Harvard Kennedy School  \nAbstract  \nReleased in 2025, Institutional Books: Harvard Library (IB-HL) is a collection of 983,004 volumes (242B O200K_BASE tokens), originally digitized through Harvard Library’s participation in the Google Books Library project. As researchers and developers have begun to use IB-HL, a tension has emerged between standard large-scale preprocessing practices and the goals of careful information stewardship. Many existing pipelines optimize for web text: as a result, they tend to aggressively filter, deduplicate, restrict by language, and sometimes discard meaningful metadata. Meanwhile, researchers seeking to use IB-HL duplicate effort while performing similar processing and analysis.  \nWe describe an approach that we call Enriched Text. Instead of producing a single “complete” stream of tokens, we normalize the text while preserving metadata through annotations. We separate endmatter, detect per-paragraph language, identify clusters of duplicate paragraphs, and compute perparagraph bits-per-byte scores. We provide this information through HTML-like annotations layered on top of the text. By parsing these annotations, users can tailor the output to their own needs instead of accepting a global editorial decision on content. The pipeline applies to all ≈250 languages in the collection.  \nThis report describes this project’s goals, implementation, and design rationale. The release includes IB-HL-ET (an enriched-text version of IB-HL containing 217B O200K_BASE tokens across 983,003 volumes, organized into 1.39B annotated subtopic paragraphs) and the pipeline that produced it. These serve to make the collection easier for machines to parse and for humans to study.  \n†[Corresponding author](Corresponding author gleppert@law.harvard.edu)[ gleppert@law.harvard.edu](Corresponding author gleppert@law.harvard.edu)  \nContents  \n1 Introduction 3  \n2 Contributions 4  \n3 Goals and Scope 4  \n4 Processing Pipeline 6  \n4.1 Preprocessing ............................................ 6  \n4.2 Unicode Normalization . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 8  \n4.3 Duplicate Page Removal ...................................... 9  \n4.4 Endmatter Separation ....................................... 9  \n4.5 Dehyphenation ............................................ 11  \n4.6 Running Header and Footer Removal ............................... 13  \n4.7 Page Number Removal ....................................... 15  \n4.8 Sentence Segmentation ....................................... 16  \n4.9 Topical Chunking .......................................... 18  \n4.10 Duplicate Identification ...................................... 19  \n4.11 Bits-Per-Byte ............................................. 24  \n4.12 Final Annotation and Formatting ................................. 26  \n5 Dataset Analysis 30  \n5.1 Tokenizability in IB-HL and IB-HL-ET ............................. 30  \n5.2 Multilingual Reuse of Duplicated Text .............................. 31  \n5.3 Bits-Per-Byte Over Time ...................................... 32  \n6 Discussion and Further Directions 33  \n7 Acknowledgements 34  \n8 Rights Determination 35  \n9 Disclaimers 35  \nReference list 36  \nAppendix A Dataset Fields 44  \nAppendix B Additional related work 44  \nAppendix C Details on Processing 45  \nC.1 Preprocessing Details ........................................ 45  \nC.2 Unicode Normalization . . . . . . . . . . . . . . . . . . . . ","cbCaim9qUziZkjwo","https://ap.wps.com/l/cbCaim9qUziZkjwo","pdf",3058866,59,"English","# Introduction\n## Contributions\n## Goals and Scope\n## Processing Pipeline\n### Preprocessing\n### Unicode Normalization\n### Duplicate Page Removal\n### Endmatter Separation\n### Dehyphenation\n### Running Header and Footer Removal\n### Page Number Removal\n### Sentence Segmentation\n### Topical Chunking\n### Duplicate Identification\n### Bits-Per-Byte\n### Final Annotation and Formatting\n# Dataset Analysis\n## Tokenizability in IB-HL and IB-HL-ET\n## Multilingual Reuse of Duplicated Text\n## Bits-Per-Byte Over Time\n# Discussion and Further Directions\n# Acknowledgements\n# Rights Determination\n# Disclaimers\n# Appendix A Dataset Fields\n# Appendix B Additional related work\n# Appendix C Details on Processing\n# Appendix D Examples with Known Limitations\n# Appendix E Using the Parser Library","[{\"question\":\"What problem does the Enriched Text approach address in IB-HL preprocessing?\",\"answer\":\"It addresses the mismatch between common web-text preprocessing and careful information stewardship, where pipelines may filter, deduplicate, restrict languages, or discard valuable metadata. Enriched Text preserves metadata via annotations while still normalizing and preparing the text at scale.\"},{\"question\":\"How does Enriched Text preserve information while normalizing OCR text?\",\"answer\":\"Instead of outputting a single complete token stream, it normalizes text while preserving metadata through HTML-like layered annotations. These annotations support user-driven tailoring rather than a single global editorial decision.\"},{\"question\":\"What processing steps are included in the pipeline?\",\"answer\":\"The pipeline includes preprocessing, Unicode normalization, duplicate page removal, endmatter separation, dehyphenation, removal of running headers/footers and page numbers, sentence segmentation, topical chunking, duplicate identification, bits-per-byte computation, and final annotation/formatting.\"}]","Institutional Books—Enriched Text - A customizable multilingual open-source pipeline for denoising, deduplicating, and annotating OCR text at scale | PDF",149]