[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-580771-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-580771-en":130},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","employing-a-hierarchical-rater-models-for-automated-scoring-scope-review-on-the-application-in-educational-assessment","Employing a Hierarchical Rater Models for Automated Scoring - Scope Review on the Application in Educational Assessment","","This scope-review traces milestones in using Hierarchical Rater Models (HRMs) to make automated essay scoring (AES) operational and to enhance instructional evaluation. Essay scoring has relied on human raters, but bias, inconsistency, and scalability concerns drive automated approaches. HRMs model rater behavior, task complexity, and interaction effects, while machine learning and NLP enable language analysis, semantic evaluation, and contextual understanding for more dependable scoring. The review also discusses integrating Signal Detection Theory to evaluate rater sensitivity and bias, improving feedback quality and formative accuracy, while noting challenges such as computational complexity, dataset availability, and algorithmic bias.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/employing-a-hierarchical-rater-models-for-automated-scoring-scope-review-on-the-application-in-educational-assessment/580771/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/employing-a-hierarchical-rater-models-for-automated-scoring-scope-review-on-the-application-in-educational-assessment/580771.png","ImageObject",300,407,{"name":42,"@type":43},"Riley","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-16","2026-10-15",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",8,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"Why are automated essay scoring systems needed in educational assessment?","Question",{"text":63,"@type":64},"Automated essay scoring addresses rater bias, inconsistency, and scalability limitations of human evaluations while supporting more reliable instructional assessment.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How do Hierarchical Rater Models (HRMs) improve automated essay scoring?",{"text":68,"@type":64},"HRMs model rater behavior and task complexity, and advances in machine learning and NLP help incorporate language analysis, semantic evaluation, and contextual understanding.",{"name":70,"@type":61,"acceptedAnswer":71},"What role does Signal Detection Theory play when integrated into HRMs?",{"text":72,"@type":64},"Integrating Signal Detection Theory helps HRMs assess rater sensitivity and bias, producing more interpretable evaluation results.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},580771,1792120192,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,111,114,119,122,126],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":108,"show_sort_weight":109,"slug":110},7,"Healthcare",40,"healthcare",{"id":55,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":112,"slug":113},30,"research-report",{"id":115,"doc_module":4,"doc_module_name":25,"category_name":116,"show_sort_weight":117,"slug":118},9,"Religion & Spirituality",20,"religion-spirituality",{"id":117,"doc_module":4,"doc_module_name":25,"category_name":120,"show_sort_weight":117,"slug":121},"World Cup","world-cup",{"id":123,"doc_module":4,"doc_module_name":25,"category_name":124,"show_sort_weight":123,"slug":125},10,"Lifestyle","lifestyle",{"id":127,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":97,"slug":129},19,"General","general",{"code":4,"msg":82,"data":131},{"doc_id":79,"user_id":132,"nickname":42,"user_avatar":133,"doc_module":4,"category_id":55,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":139,"language":140,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":12,"update_tm":144,"read_time":145,"preview_page_count":4},1374391975076,"https://ap-avatar.wpscdn.com/avatar/14000253ca4ec9f6853?x-image-process=image/resize,m_fixed,w_180,h_180&k=1783305029341752051","Employing a Hierarchical Rater Models for Automated Scoring: Scope Review on the Application in Educational Assessment  \nAkif AVCU [1]  \nTo Cite: Avcu, A. (2025) . Employing a hierarchical rater models for automated scoring: Scope review on the application in educational assessment. Malaysian Online Journal of Educational Technology, 13 (2), 37-47.  \n[http://dx.doi.org/10.52380/mojet.2025.13.2.569](http://dx.doi.org/10.52380/mojet.2025.13.2.569)  \nABSTRACT  \n\n|  |  |  |\n| --- | --- | --- |\n|  |  |  |\n| This scope-review presents the milestones of how Hierarchical Rater Models (HRMs) become operable to used in automated essay scoring (AES) to improve instructional evaluation. Although essay evaluations—a useful instrument for evaluating higher-order cognitive abilities—have always depended on human raters, concerns regarding rater bias, inconsistency, and scalability have motivated the development of automated systems. By modeling rater behaviors, task complexity, and interaction effects, HRMs handle these issues and offer a strong base to minimize biases and increase dependability. Advances in machine learning and natural language processing (NLP) have helped HRMs to be included into AES systems. Leveraging HRMs to rectify rater biases and guarantee fairness, these systems include language analysis, semantic evaluation, and contextual understanding. This review also includes how Signal Detection Theory (SDT) is being included into HRMs to improve their capacity to assess rater sensitivity and bias and provide understandable results. By including HRMs into AES, feedback quality is improved as well as score accuracy, hence facilitating more exact formative evaluations become possible. Still present, though, are difficulties like computing complexity, dataset availability, and algorithmic bias. The paper emphasizes the possibilities of HRMs in creating fair, high-quality, scalable evaluation systems and supports ongoing research to improve these approaches for various educational environments. |  |  |\n|  |  |  |\n| Keywords: | Hierarchical review | rater model,; automated scoring; essays; Scope |\n\n[1] [avcuakif@gmail.com](avcuakif@gmail.com), [orcid.org/ 0000-0003-1977-7592](orcid.org/ 0000-0003-1977-7592), Marmara University, Türkiye  \nArticle History:  \nReceived: 4 August 2024  \nReceived in revised form: 14 Oct.. 2024  \nAccepted: 30 November 2024  \nArticle type: Review Article  \nINTRODUCTION  \nTransition of essay evaluation from human raters to automated systems  \nThe assessment of student learning is critically important in the rapidly evolving educational settings. The qualith of evaluation for learning outcomes had been done via tests and examinations and can be enhanced by diverse question formats and grading techniques. Question types can be formulated to include several formats, ranging from basic multiple-choice questions to those requiringhigher-order cognitive skills, such as essays and other kinds of complex tasks. Traditional evaluation methods, such multiple-choice and true-false examinations, prioritize recaling and recognizing but sometimes cannot catch deeper cognitive skills (Simkin & Kuechler, 2005) . On the other hand, essay examinations are especially effective in assessing a broader spectrum of competencies, such as critical thinking skills and reasoning. Essay tests facilitate  \nstudents' engagement in higher-order cognitive processes. Such evaluations require the application, analysis, synthesis, and appraisal levels of cognitive skills (Bloom, 1956; Anderson et al., 2001) .  \nEssay questions necessitate that students synthesize information from diverse sources, draw connections among concepts, and exhibit a thorough comprehension of the subject (Biggs & Tang, 2011) . According to Ramsden (1992), when students interact with knowledge meaningfully rather than simply memorizing it, a higher order knowledge occur when they take exams with essays. In their inspirational study on learning methodologies, Marton a","cbCaijwjmsnnuJjC","https://ap.wps.com/l/cbCaijwjmsnnuJjC","pdf",368805,11,"English","# Abstract\n# Introduction\n## Transition of essay evaluation from human raters to automated systems\n# Keywords","[{\"question\":\"Why are automated essay scoring systems needed in educational assessment?\",\"answer\":\"Automated essay scoring addresses rater bias, inconsistency, and scalability limitations of human evaluations while supporting more reliable instructional assessment.\"},{\"question\":\"How do Hierarchical Rater Models (HRMs) improve automated essay scoring?\",\"answer\":\"HRMs model rater behavior and task complexity, and advances in machine learning and NLP help incorporate language analysis, semantic evaluation, and contextual understanding.\"},{\"question\":\"What role does Signal Detection Theory play when integrated into HRMs?\",\"answer\":\"Integrating Signal Detection Theory helps HRMs assess rater sensitivity and bias, producing more interpretable evaluation results.\"}]","Employing a Hierarchical Rater Models for Automated Scoring - Scope Review on the Application in Educational Assessment | PDF",1792038262,28]