[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-273619-105":59,"doc-detail-273619-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","institutional-newspapers-pipeline-deriving-billions-of-high-quality-tokens-from-historical-newspapers","Institutional Newspapers Pipeline - Deriving Billions of High Quality Tokens from Historical Newspapers","","Historical newspapers provide an extensive record of public life, yet their dense, irregular, and sometimes noisy page layouts make computational access difficult and limited. The Institutional Newspapers Pipeline extracts high-quality, structured datasets from historical newspaper scans through an interpretable, customizable, and workstation-feasible workflow. Each scan is processed via segmentation, OCR, text analysis, type classification, reading order detection, named entity recognition, subject classification, language detection, and pre-computed embeddings generation. Evaluations and dataset-scale measurements accompany the released pipeline, models, and open dataset.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/institutional-newspapers-pipeline-deriving-billions-of-high-quality-tokens-from-historical-newspapers/273619/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/institutional-newspapers-pipeline-deriving-billions-of-high-quality-tokens-from-historical-newspapers/273619.png","ImageObject",300,407,{"name":92,"@type":93},"Asher","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-25","2026-09-15",true,{"@type":102,"interactionType":103,"userInteractionCount":39},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What is the Institutional Newspapers Pipeline designed to do?","Question",{"text":112,"@type":113},"It extracts high-quality, structured datasets from historical newspaper scans by transforming each scan into interpretable crop-level outputs and analytics products.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does the pipeline process each newspaper scan?",{"text":117,"@type":113},"It segments scans into type-agnostic crops, runs OCR per crop, then performs text analysis, type classification, reading order detection, named entity recognition, subject classification, language detection, and embeddings generation.",{"name":119,"@type":110,"acceptedAnswer":120},"What scale of data did the authors generate and release?",{"text":121,"@type":113},"They ran the pipeline on a portion of Boston Public Library holdings and released an open dataset with 16.3 billion o200k_base tokens across 83.1 million crops from 1,473,635 public domain scans published between 1795 and 1930.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},273619,1789472294,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":39,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":129,"read_time":144,"preview_page_count":4},687197207639,"https://ap-avatar.wpscdn.com/davatar_a8503ba1806abce46bf441b54a3ca4cd","Institutional Newspapers Pipeline: Deriving billions of high quality tokens from historical newspapers  \nMatteo Cargnelutti ᵃ, Catherine Brobston ᵃ, Eben English ᵇ, Jake Sadow ᵇ, Kacie Bailey ᵃ, Greg Leppert ✉ ᵃ, Amanda Watson ᶜ, Jessica Chapel ᵇ, Jonathan Zittrain ᵈ  \nᵃ Institutional Data Initiative, Harvard Law School Library ᵇ Boston Public Library  \nᶜ Harvard Law School Library  \nᵈ Harvard Law School, Harvard School of Engineering and Applied Sciences, Harvard Kennedy School  \nAbstract  \nHistorical newspapers are an abundant record of public life, but their dense, irregular and sometimes noisy layouts make computational access to these materials both challenging and limited. We present the Institutional Newspapers Pipeline, a modular system we jointly designed with Boston Public Library to extract high-quality, structured datasets from historical newspaper scans. It was architected so that each step remains interpretable and customizable, and so that the pipeline as a whole remains computationally frugal enough to run on workstation-level hardware. The pipeline runs each scan through a multi-step process: it segments scans into individual type-agnostic crops and performs OCR on each resulting segment before then performing text analysis, type classification, reading order detection, named entities recognition, subject classification, language detection, and pre-computed embeddings generation on every crop. We ran this pipeline against a portion of Boston Public Library's holdings and released the results as an open dataset. The optical character recognition (OCR) output represents 16.3 billion o200k_base tokens across 83.1 million individual crops, extracted from 1,473,635 public domain newspaper scans published between 1795 and 1930. This report describes our methods for each processing step, the small models we trained, as well as the evaluation results and dataset-scale measurements we collected in the  \nprocess. It accompanies the release of the pipeline, models, and dataset. We position this work as a substantial step towards unlocking high-quality data from tens of millions of newspaper scans.  \n✉ Corresponding author: [gleppert@law.harvard.edu](gleppert@law.harvard.edu)  \n1 Introduction......................................................................................................................................................... 2  \n2 Contributions.......................................................................................................................................................4  \n3 Segmentation.......................................................................................................................................................4  \n3.1 Methodology............................................................................................................................................. 4  \n3.2 Results....................................................................................................................................................... 6  \n4 Crop-level OCR.................................................................................................................................................. 7  \n4.1 Methodology............................................................................................................................................. 7  \n4.2 Results....................................................................................................................................................... 9  \n5 Crop-level Language Detection and Text Analysis.............................................................................................9  \n5.1 Methodology............................................................................................................................................. 9  \n5.2 Results..............................................................................................................................","cbCaifbC11OvADj2","https://ap.wps.com/l/cbCaifbC11OvADj2","pdf",6219365,37,"English","# 1 Introduction\n# 2 Contributions\n# 3 Segmentation\n## 3.1 Methodology\n## 3.2 Results\n# 4 Crop-level OCR\n## 4.1 Methodology\n## 4.2 Results\n# 5 Crop-level Language Detection and Text Analysis\n## 5.1 Methodology\n## 5.2 Results\n# 6 Crop-type Classification\n## 6.1 Methodology\n## 6.2 Results\n# 7 Scan-level Reading Order Detection\n## 7.1 Methodology\n## 7.2 Results\n# 8 Crop-level Named Entity Recognition\n## 8.1 Methodology\n## 8.2 Results\n# 9 Crop-level Subject Detection\n## 9.1 Methodology\n## 9.2 Results\n# 10 Crop-level Embeddings","[{\"question\":\"What is the Institutional Newspapers Pipeline designed to do?\",\"answer\":\"It extracts high-quality, structured datasets from historical newspaper scans by transforming each scan into interpretable crop-level outputs and analytics products.\"},{\"question\":\"How does the pipeline process each newspaper scan?\",\"answer\":\"It segments scans into type-agnostic crops, runs OCR per crop, then performs text analysis, type classification, reading order detection, named entity recognition, subject classification, language detection, and embeddings generation.\"},{\"question\":\"What scale of data did the authors generate and release?\",\"answer\":\"They ran the pipeline on a portion of Boston Public Library holdings and released an open dataset with 16.3 billion o200k_base tokens across 83.1 million crops from 1,473,635 public domain scans published between 1795 and 1930.\"}]","Institutional Newspapers Pipeline - Deriving Billions of High Quality Tokens from Historical Newspapers | PDF",93]