[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-158017-105":59,"doc-detail-158017-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","bert-irt-accelerating-item-piloting-with-bert-embeddings-and-explainable-irt-models-improve-efficiency","BERT-IRT - Accelerating Item Piloting with BERT Embeddings and Explainable IRT Models - 提升效率","","Estimating item parameters such as question difficulty is essential for high-stakes assessments but typically depends on lengthy pilots that collect response data from representative test-takers. These pilots constrain item bank growth and refresh frequency, increase assessment costs, and consume valuable testing time. This paper introduces an explanatory IRT model, BERT-IRT, applied to the Duolingo English Test to reduce pilot length by a factor of 10. Evaluation shows accelerated item piloting using BERT embeddings and engineered NLP features without sacrificing criterion validity or reliability.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/bert-irt-accelerating-item-piloting-with-bert-embeddings-and-explainable-irt-models-improve-efficiency/158017/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/bert-irt-accelerating-item-piloting-with-bert-embeddings-and-explainable-irt-models-improve-efficiency/158017.png","ImageObject",300,407,{"name":92,"@type":93},"Valentina","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-28","2026-08-29",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",17,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"Why are item piloting and large response datasets needed in high-stakes tests?","Question",{"text":113,"@type":114},"Item parameters typically require hundreds of responses per item collected via pilots. Pilots ensure items are estimated accurately enough to support reliable item selection and testing quality.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"What does BERT-IRT aim to improve compared with conventional item parameter estimation?",{"text":118,"@type":114},"BERT-IRT uses BERT embeddings and engineered NLP features within an explanatory IRT framework to reduce pilot length substantially while keeping performance measures intact.",{"name":120,"@type":111,"acceptedAnswer":121},"How does BERT-IRT maintain validity and reliability while accelerating item piloting?",{"text":122,"@type":114},"The evaluation reports that the model accelerates item piloting by leveraging BERT representations and engineered features without sacrificing criterion validity or reliability.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},158017,1787993644,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":130,"read_time":145,"preview_page_count":4},13056703020460,"https://ap-avatar.wpscdn.com/avatar/be000253dac470eee5d?_k=1778207105932848923","BERT-IRT: Accelerating Item Piloting with BERT Embeddings and  \nExplainable IRT Models  \nKevin P. Yancey and Andrew Runge and Geoff LaFlair and Phoebe Mulcaire  \nDuolingo  \n5900 Penn Ave  \nPittsburgh, PA 15206  \n{kyancey,arunge,geoff,[phoebe}@duolingo.com](phoebe}@duolingo.com)  \nAbstract  \nEstimating item parameters (e.g., the difficulty of a question) is an important part of modern high-stakes tests. Conventional methods require lengthy pilots to collect response data from a representative population of test-takers.  \nThe need for these pilots limit item bank size and how often those item banks can be refreshed, impacting test security, while increasing costs needed to support the test and taking up the test-taker’s valuable time. Our paper presents a novel explanatory item response theory (IRT) model, BERT-IRT, that has been used on the Duolingo English Test (DET), a highstakes test of English, to reduce the length of pilots by a factor of 10 . Our evaluation showshow the model uses BERT embeddings and engineered NLP features to accelerate item piloting without sacrificing criterion validity or reliability.  \n1 Introduction  \nThe Duolingo English Test (DET) is a test of English language proficiency that is used for admissions decisions in English medium universities. It measures the four skills of speaking, writing, reading, and listening. It is delivered remotely to testtakers’ computers via a desktop application, and it can be taken any time and at any appropriate location with a strong enough internet connection. The DET’s value proposition to test-takers is that it is affordable, short in duration, and has a short score reporting turn-around time.  \nThe DET accomplishes this, in part, by using computer adaptive test (CAT) administration to more quickly and accurately estimate test-takers’language proficiency (Cardwell et al., 2022) . A computer adaptive test (CAT) uses item parameter estimates to adapt to each test-taker by finding items that will yield maximal information about their proficiency based on how well they’ve done so far. Item banks for CATs must be very large to  \nensure that test-takers do not have preknowledge of items (LaFlair et al., 2022 ; Way, 1998), and they also require high-quality item parameter estimates to ensure that items are selected for administration accurately.  \nTypically, item parameters are estimated from hundreds of responses for each item collected via pilots. However, these pilots take up the test-taker’s valuable time and increase the costs for the assessment, thus limiting the rate at which new items can be added to the bank. Explanatory frameworks that estimate item parameters from item features have been around for a long time, starting with Fischer (1973)’s Log Linear Traits Model (LLTM), and have a rich literature (De Boeck, 2004) . These frameworks can be used to help reduce or eliminate the need for item piloting by leveraging item features to estimate item parameters more accurately with less response data. This can have positive downstream effects on test security and on testtakers. For security, it allows for test developers to add to, or replace, their item banks at very high rates, which helps to ensure unique administrations of tests and reduce the effects of item preknowledge. For test-takers, it reduces the amount of time they spend responding to unscored test items during pilots and reduces the costs oftest development. These cost savings can be passed on to test-takers and even help lower barriers for less economically advantaged test-takers.  \nIt is well known in the NLP literature (Tenney et al., 2019 ; Jawahar et al., 2019) that pretrained language models such as BERT (Devlinet al., 2019) learn text representations that represent highly general linguistic properties of words that are useful for a wide range of tasks, including estimating the difficulty of text for L2 learners (Yancey et al., 2021) . More recent work has explored using these text embeddings in ex","cbCaihhZEMGZTKDY","https://ap.wps.com/l/cbCaihhZEMGZTKDY","pdf",558944,11,"English","# Introduction\n## Duolingo English Test and CAT\n## Item parameter estimation and limitations of pilots\n## Explanatory IRT and pretrained language model embeddings\n## Related work and prior BERT-LLTM approach\n## Improvements introduced by BERT-IRT","[{\"question\":\"Why are item piloting and large response datasets needed in high-stakes tests?\",\"answer\":\"Item parameters typically require hundreds of responses per item collected via pilots. Pilots ensure items are estimated accurately enough to support reliable item selection and testing quality.\"},{\"question\":\"What does BERT-IRT aim to improve compared with conventional item parameter estimation?\",\"answer\":\"BERT-IRT uses BERT embeddings and engineered NLP features within an explanatory IRT framework to reduce pilot length substantially while keeping performance measures intact.\"},{\"question\":\"How does BERT-IRT maintain validity and reliability while accelerating item piloting?\",\"answer\":\"The evaluation reports that the model accelerates item piloting by leveraging BERT representations and engineered features without sacrificing criterion validity or reliability.\"}]","BERT-IRT - Accelerating Item Piloting with BERT Embeddings and Explainable IRT Models - 提升效率 | PDF",28]