[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-493063-105":59,"doc-detail-493063-en":122},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":115,"head_meta":117,"extra_data":119,"updated_unix":121},105,"en","dirt-cheap-web-scale-parallel-text-from-the-common-crawl-research-paper","Dirt Cheap Web-Scale Parallel Text from the Common Crawl - Research Paper","","Parallel text is essential for modern machine translation, but broad web crawling is infeasible for most researchers and companies. This work mines web-scale parallel data from the public Common Crawl, using an open-source extension of the STRAND algorithm. Starting only from two-letter language codes, the authors process 32TB in under a day at low cost, extracting parallel corpora across many languages and domains. The mined data boosts translation quality across multiple pairs, with improvements up to 5 BLEU, and is released for further research.",{"@graph":69,"@context":114},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/dirt-cheap-web-scale-parallel-text-from-the-common-crawl-research-paper/493063/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/dirt-cheap-web-scale-parallel-text-from-the-common-crawl-research-paper/493063.png","ImageObject",300,407,{"name":92,"@type":93},"xXChrisXx","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-11","2026-10-01",true,{"@type":102,"interactionType":103,"userInteractionCount":44},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108],{"name":109,"@type":110,"acceptedAnswer":111},"How do the authors find candidate parallel document pairs?","Question",{"text":112,"@type":113},"They use an extension of the STRAND algorithm that identifies potentially parallel documents using cues from URLs and document content.","Answer","https://schema.org",{"og:url":83,"og:type":116,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":118,"canonical":83},"index,follow",{"doc_id":120,"site_id":62},493063,1791077113,{"code":4,"msg":5,"data":123},{"doc_id":120,"user_id":124,"nickname":92,"user_avatar":125,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":126,"file_id":127,"file_url":128,"file_type":129,"file_size":130,"view_count":44,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":52,"language":131,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":132,"faqs":133,"seo_title":134,"seo_description":67,"update_tm":135,"read_time":136},8796112263567,"https://ap-avatar.wpscdn.com/davatar_3d24733baf745e90a7e4bdd5f77d97b2","Dirt Cheap Web-Scale Parallel Text from the Common Crawl  \nJason R. Smith 1 ,2 [jsmith@cs.jhu.edu](jsmith@cs.jhu.edu)  \nPhilipp Koehn3 [pkoehn@inf.ed.ac.uk](pkoehn@inf.ed.ac.uk)  \nHerve Saint-Amand3 [herve@saintamh.org](herve@saintamh.org)  \nChris Callison-Burch 1 ,2 ,5 [ccb@cs.jhu.edu](ccb@cs.jhu.edu) ⇤  \nMagdalena Plamada4 [plamada@cl.uzh.ch](plamada@cl.uzh.ch)[ ](plamada@cl.uzh.ch)Adam Lopez 1 ,2 [alopez@cs.jhu.edu](alopez@cs.jhu.edu)  \n1Department of Computer Science, Johns Hopkins University  \n2Human Language Technology Center of Excellence, Johns Hopkins University  \n3 School of Informatics, University of Edinburgh  \n4Institute of Computational Linguistics, University of Zurich  \n5 Computer and Information Science Department, University of Pennsylvania  \nAbstract  \nParallel text is the fuel that drives modern machine translation systems. The Web is a comprehensive source of preexisting parallel text, but crawling the entire web is impossible for all but the largest companies. We bring web-scale parallel text to the masses by mining the Common Crawl, a public Web crawl hosted on Amazon’s Elastic Cloud. Starting from nothing more than a set of common two-letter language codes, our open-source extension of the STRAND algorithm mined 32 terabytes of the crawl in just under a day, at a cost of about $500 . Our large-scale experiment uncovers large amounts of parallel text in dozens of language pairs across a variety of domains and genres, some previously unavailable in curated datasets. Even with minimal cleaning and filtering, the resulting data boosts translation performance across the board for five different language pairs in the news domain, and on open domain test sets we see improvements of up to 5 BLEU. We make our code and data available for other researchers seeking to mine this rich new data resource.1  \n1 Introduction  \nA key bottleneck in porting statistical machine translation (SMT) technology to new languages and domains is the lack of readily available parallel corpora beyond curated datasets. For a handful of language pairs, large amounts of parallel data  \n⇤ This research was conducted while Chris CallisonBurch was at Johns Hopkins University.  \n[1](1github.com/jrs026/CommonCrawlMiner)[github.com/jrs026/CommonCrawlMiner](1github.com/jrs026/CommonCrawlMiner)  \nare readily available, ordering in the hundreds of millions of words for Chinese-English and ArabicEnglish, and in tens of millions of words for many European languages (Koehn, 2005) . In each case, much of this data consists of government and news text. However, for most language pairs and domains there is little to no curated parallel data available. Hence discovery of parallel data is an important first step for translation between most of the world’s languages.  \nThe Web is an important source of parallel text. Many websites are available in multiple languages, and unlike other potential sources—such as multilingual news feeds (Munteanu and Marcu, 2005) or Wikipedia (Smith et al., 2010)—it is common to find document pairs that are direct translations of one another. This natural parallelism simplifies the mining task, since few resources or existing corpora are needed at the outset to bootstrap the extraction process.  \nParallel text mining from the Web was originally explored by individuals or small groups of academic researchers using search engines (Nie et al., 1999; Chen and Nie, 2000; Resnik, 1999; Resnik and Smith, 2003) . However, anything more sophisticated generally requires direct access to web-crawled documents themselves along with the computing power to process them. For most researchers, this is prohibitively expensive. As a consequence, web-mined parallel text has become the exclusive purview of large companies with the computational resources to crawl, store, and process the entire Web.  \nTo put web-mined parallel text back in the hands of individual researchers, we mine parallel text from the Common Crawl, a regularly updated 81-terabyte","cbCaidVEKhv8HeYI","https://ap.wps.com/l/cbCaidVEKhv8HeYI","pdf",189506,"English","# Abstract\n# Introduction\n# Mining the Common Crawl\n## Candidate pair selection","[{\"question\":\"How do the authors find candidate parallel document pairs?\",\"answer\":\"They use an extension of the STRAND algorithm that identifies potentially parallel documents using cues from URLs and document content.\"}]","Dirt Cheap Web-Scale Parallel Text from the Common Crawl - Research Paper | PDF",1790874449,25]