[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-530013-105":59,"doc-detail-530013-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","crase-essay-scoring-model-performance-based-on-proof-of-concept-and-operational-engine-trainings-data-byte","CRASE® Essay Scoring Model Performance Based on Proof-of-Concept and Operational Engine Trainings - Data Byte","","CRASE® essay scoring models have been trained on hundreds of constructed-response items and essay prompts since 2007, supporting formative, summative, and research assessments across multiple content areas and grades 4–12. A Data Byte summarizes human–CRASE scoring metrics observed from 2016–2019, focusing on blind-validation automated metrics compared to human hand-scoring. It reviews two evaluation metrics—quadratic weighted kappa and exact agreement rate—then reports overall model performance and discusses factors that can influence scoring accuracy, including training parameters and prompt-dimension suitability.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/crase-essay-scoring-model-performance-based-on-proof-of-concept-and-operational-engine-trainings-data-byte/530013/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/crase-essay-scoring-model-performance-based-on-proof-of-concept-and-operational-engine-trainings-data-byte/530013.png","ImageObject",300,407,{"name":92,"@type":93},"ahduag","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-10","2026-10-09",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",11,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What does the paper mean by human–CRASE scoring metrics?","Question",{"text":113,"@type":114},"Human–CRASE metrics are automated scoring estimates evaluated against corresponding human hand-scoring metrics using a blind-validation sample not used for training.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"Which automated metrics are used to evaluate CRASE essay scoring models?",{"text":118,"@type":114},"The paper evaluates quadratic weighted kappa (QWK) and exact agreement rate, comparing human–CRASE results with human–human hand-scoring outcomes.",{"name":120,"@type":111,"acceptedAnswer":121},"How did CRASE performance compare to industry standards between 2016 and 2019?",{"text":122,"@type":114},"Across prompt-dimension combinations, 85% met or exceeded benchmarks for exact agreement rate and 92% met or exceeded benchmarks for QWK.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},530013,1791689499,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":29,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":145,"preview_page_count":19,"preview_urls":146},3985747870414,"https://ap-avatar.wpscdn.com/davatar_9964176cb1d06d4a9deccf72a44ae3dc","CRASE® Essay Scoring Model Performance Based on Proof-of-Concept and Operational Engine Trainings  \nScott W. Wood  \nSince the creation of CRASE® in 2007, the CRASE Research team has trained the engine on hundreds of constructed-response items and essay prompts. These scoring models have been used in formative, summative, and research assessments across a variety of content areas—English language arts, mathematics, science, government—and for grade levels ranging from 4th to 12th grade. Most models have been prompt-specific, meaning that hand-scored response data for a single item were used to train the model, though ACT also has experience working with cross-prompt scoring models—that is, models trained on multiple similar essay prompts simultaneously.  \nThis Data Byte presents an overview of the human-CRASE scoring metrics that ACT observed between 2016 and 2019 on writing prompts. The next section describes the two evaluation metrics considered in this paper. This is followed by a summary of CRASE essay scoring performance. The Data Byte concludes with commentary about factors that may affect essay scoring accuracy.  \nAutomated Scoring Metrics  \nAfter training a scoring model in CRASE, a blind-validation sample (i.e. , a sample of responses not used for training) will be scored. This sample is used to calculate automated scoring metrics that estimate how the scoring model would be expected to perform on new responses during operational scoring. This research focuses on human-CRASE metrics evaluated relative to corresponding human-human metrics from hand scoring.  \nQuadratic weighted kappa (QWK) is a popular metric used in the automated scoring literature. QWK is a measure of rater agreement that penalizes for disagreement between two independent raters. Small differences between the raters lead to small penalties; larger differences lead to larger penalties. QWK can be thought of as a slight variation of a Pearson correlation, as the two metrics will often be within .01 of each other in practice.  \nIf two independent human raters obtain a certain QWK, the goal is to achieve a similar humanCRASE QWK. Industry-standard criteria suggest that, if the human-CRASE QWK is less than the human-human QWK, then the human-CRASE QWK should be within .10 of the humanhuman QWK (Williamson et al. , 2012) .  \n© 2023 by ACT, Inc. All rights reserved. | R2304  \nSome stakeholders prefer the exact agreement rate due to its simplicity. The exact agreement rate is the percentage of essays for which two independent raters assign the same score. The goal is to achieve a human-CRASE exact agreement rate similar to the human-human exact agreement rate. Industry-standard criteria suggest that, if the human-CRASE exact agreement rate is less than the human-human exact agreement rate, then the human-CRASE exact agreement rate should be within 5.125 percentage points of the human-human exact agreement rate ( McGraw-Hill Education CTB , 2014) .  \nCRASE and Essay Scoring  \nBetween 2016 and 2019, the CRASE Research team trained models for 253 essay promptdimension combinations across multiple assessment programs. (For prompts scored on multiple dimensions, an independent model is produced for each dimension.) Of these 253 promptdimension combinations, human-human scoring metrics were available for 165 (the remaining combinations included only a single human rater) . The human-human and human-CRASE exact agreement rates for these 165 prompt-dimension combinations are plotted in Figure 1.  \nFigure 1. Human-Human and Human-CRASE Exact Agreement Rates, by Prompt-Dimension  \nNote that there are two reference lines in the plot. The dashed line marks where the humanhuman exact agreement rates are equal to the human-CRASE exact agreement rates. The solid line marks where the human-CRASE exact agreement rates are 5.125 percentage points less  \nthan the human-human exact agreement rates. To meet industry standards regarding exact agreement rate, a prompt-dimension poi","cbCaiqSuYcA1FJ34","https://ap.wps.com/l/cbCaiqSuYcA1FJ34","pdf",149390,"English","# Automated Scoring Metrics\n## Quadratic Weighted Kappa (QWK)\n## Exact Agreement Rate\n# CRASE and Essay Scoring\n## Human-CRASE vs Human-Human Metrics\n# Factors that Affect Scoring Accuracy","[{\"question\":\"What does the paper mean by human–CRASE scoring metrics?\",\"answer\":\"Human–CRASE metrics are automated scoring estimates evaluated against corresponding human hand-scoring metrics using a blind-validation sample not used for training.\"},{\"question\":\"Which automated metrics are used to evaluate CRASE essay scoring models?\",\"answer\":\"The paper evaluates quadratic weighted kappa (QWK) and exact agreement rate, comparing human–CRASE results with human–human hand-scoring outcomes.\"},{\"question\":\"How did CRASE performance compare to industry standards between 2016 and 2019?\",\"answer\":\"Across prompt-dimension combinations, 85% met or exceeded benchmarks for exact agreement rate and 92% met or exceeded benchmarks for QWK.\"}]","CRASE® Essay Scoring Model Performance Based on Proof-of-Concept and Operational Engine Trainings - Data Byte | PDF",1791522398,15,[147,151,155,159],{"page_no":8,"url":148,"width":149,"height":150},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/9ac5d6616c2576bf4b428328ed63be83.webp",983,1272,{"page_no":14,"url":152,"width":153,"height":154},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/cf717937f0afb740abc3a600c5b71e74.webp",982,1270,{"page_no":81,"url":156,"width":157,"height":158},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/d99fdfa3acb18ff762a2614f89a34f75.webp",981,1269,{"page_no":19,"url":160,"width":161,"height":162},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/f77215ddd00a5a7950d1f73ee201e1d0.webp",980,1268]