[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-426984-105":59,"doc-detail-426984-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","the-jhu-submission-to-the-2020-duolingo-shared-task-on-simultaneous-translation-and-paraphrase-for-language-education","The JHU Submission to the 2020 Duolingo Shared Task on Simultaneous Translation and Paraphrase for Language Education","","Johns Hopkins University presents its submission to the 2020 Duolingo STAPLE Shared Task for language education, covering all five translation directions and ranking first for every language pair. The system uses a language-agnostic three-stage pipeline: train general-domain machine translation, finetune on Duolingo-provided data, then produce and filter n-best hypotheses with score-based methods. Linguistically motivated variants were tested with limited impact, and stronger BLEU for beam-search outputs did not guarantee better weighted macro F1 on the reference-recovery metric.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/the-jhu-submission-to-the-2020-duolingo-shared-task-on-simultaneous-translation-and-paraphrase-for-language-education/426984/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/the-jhu-submission-to-the-2020-duolingo-shared-task-on-simultaneous-translation-and-paraphrase-for-language-education/426984.png","ImageObject",300,407,{"name":92,"@type":93},"Rhys","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-22","2026-09-29",true,{"@type":102,"interactionType":103,"userInteractionCount":29},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What is the STAPLE shared task and which languages does it cover?","Question",{"text":112,"@type":113},"The STAPLE task generates a comprehensive set of translations for an English sentence into Hungarian, Japanese, Korean, Portuguese, and Vietnamese, using a list-recovery formulation rather than a single-reference BLEU comparison.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does the proposed approach select final translation candidates?",{"text":117,"@type":113},"It builds strong machine translation systems, finetunes on Duolingo-provided data, generates large n-best lists, and filters them using various score-based techniques to form the final candidate list.",{"name":119,"@type":110,"acceptedAnswer":120},"Why is BLEU not a reliable indicator of performance for this task?",{"text":121,"@type":113},"The paper finds that improving BLEU for beam-search generated translations does not necessarily improve the official task metric, weighted macro F1, which is computed over the set of hypotheses against weighted references.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},426984,1790716333,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":29,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":52,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":143,"read_time":144},687207024643,"https://ap-avatar.wpscdn.com/davatar_3d24733baf745e90a7e4bdd5f77d97b2","The JHU Submission to the 2020 Duolingo Shared Task on Simultaneous Translation and Paraphrase for Language Education  \nHuda Khayrallahz Jacob Bremermanx Arya D. McCarthyz  \nKenton Murrayz Winston Wuz and Matt Postz,yz Center for Language and Speech Processing, Johns Hopkins University yHuman Language Technology Center of Excellence, Johns Hopkins University xUniversity of Maryland, College Park  \nAbstract  \nThis paper presents the Johns Hopkins University submission to the 2020 Duolingo Shared Task on Simultaneous Translation and Paraphrase for Language Education (STAPLE) . We participated in all five language tasks, plac  \ning first in each. Our approach involved a language-agnostic pipeline of three components: (1) building strong machine translation systems on general-domain data,(2) finetuning on Duolingo-provided data, and (3) generating n-best lists which are then filtered with various score-based techniques. In addition to the language-agnostic pipeline, we attempted a number of linguistically-motivated approaches, with, unfortunately, little success. We also find that improving BLEU performance of the beam-search generated translation does not necessarily improve on the task metric􀀖weighted macro F1 of an n-best list.  \n1 Introduction  \nThe Duolingo 2020 STAPLE Shared Task (Mayhew et al., 2020) focuses on generating a comprehensive set of translations for a given sentence, translating from English into Hungarian, Japanese, Korean, Portuguese, and Vietnamese. The formulation of this task (x2) differs from the conventional machine translation setup: instead of the n-gram match (BLEU) against a single reference, sentence-level exact match is computed between a list of proposed candidates and a weighted list of references (as in Figure 1) . The set of references is drawn from Duolingo's language-teaching app. Any auxiliary data is allowed for building systems, including existing very-large parallel corpora for translation.  \nOur approach begins with strong MT systems (x3) which are fine-tuned on Duolingo-provided data (x4) . We then generate large n-best lists, from which we select our final candidate list (x5) . Our  \n\n| can i walk there? |  | eu posso andar lá? | 0.049 |\n| --- | --- | --- | --- |\n|  |  |  |  |\n\n| posso andar lá? | 0.015 |\n| --- | --- |\n\n| dá para eu andar ali? | 0.001 |\n| --- | --- |\n\n| eu posso andar pra lá? | 0.001 |\n| --- | --- |\n\nFigure 1: An example English source sentence with its weighted Portuguese target translations. The objective of the task is to recover the list of references, and performance is measured by a weighted F-score.  \nentries outperform baseline weighted F1 scores by a factor of 2 to 10 and are ranked first in the official evaluation for every language pair (x6.2) .  \nIn addition to our system description, we perform additional analysis (x7) . We find that stronger BLEU performance of the beam-search generated translation is not indicative of improvements on the task metric􀀖weighted macro F1 of a set of hypotheses􀀖and suggest this should encourage further research on how to train NMT models when n-best lists are needed (x7.1) . We perform detailed analysis on our output (x7.2), which led to additional development on English􀀕 Portuguese (x8.1) . We also present additional linguistically-informed methods which we experimented with but which ultimately did not improve performance (x8) .  \n2 Task Description  \nData We use data provided by the STAPLE shared task (Mayhew et al., 2020) . This data consists of a single English prompt sentence or phrase paired with multiple translations in the target lan-  \n\n|  | hu | ja | ko | pt | vi |\n| --- | --- | --- | --- | --- | --- |\n| total prompts | 4,000 | 2,500 | 2,500 | 4,000 | 3,500 |\n| mean translations | 63 | 342 | 280 | 132 | 56 |\n| median translations | 36 | 192 | 154 | 68 | 30 |\n| STD. translations | 66 | 362 | 311 | 150 | 62 |\n\nTable 1: Statistics over the Duolingo-provided data.  \nguage. These translations come from courses intended t","cbCaia1N59JW2erM","https://ap.wps.com/l/cbCaia1N59JW2erM","pdf",330270,"English","# Abstract\n# 1 Introduction\n# 2 Task Description\n## Data\n## Evaluation metric\n# 3 Machine Translation Systems","[{\"question\":\"What is the STAPLE shared task and which languages does it cover?\",\"answer\":\"The STAPLE task generates a comprehensive set of translations for an English sentence into Hungarian, Japanese, Korean, Portuguese, and Vietnamese, using a list-recovery formulation rather than a single-reference BLEU comparison.\"},{\"question\":\"How does the proposed approach select final translation candidates?\",\"answer\":\"It builds strong machine translation systems, finetunes on Duolingo-provided data, generates large n-best lists, and filters them using various score-based techniques to form the final candidate list.\"},{\"question\":\"Why is BLEU not a reliable indicator of performance for this task?\",\"answer\":\"The paper finds that improving BLEU for beam-search generated translations does not necessarily improve the official task metric, weighted macro F1, which is computed over the set of hypotheses against weighted references.\"}]","The JHU Submission to the 2020 Duolingo Shared Task on Simultaneous Translation and Paraphrase for Language Education | PDF",1790642420,25]