[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-582800-105":59,"doc-detail-582800-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","examining-inter-rater-reliability-of-evaluators-judging-teacher-performance-proposing-an-alternative-to-cohens-kappa","Examining Inter-Rater Reliability of Evaluators Judging Teacher Performance - Proposing an Alternative to Cohen’s Kappa","","The validity of chance-corrected agreement indices based on Cohen’s Kappa has been challenged when rating-scale category prevalence is low and rater agreement is high. This study proposes the Lambda Coefficient of Rater-Mediated Agreement as an alternative, correcting for chance agreement under assumptions about raters and the rater-mediated process. Results from teacher profile ratings show Kappa’s shortcomings and demonstrate the robustness of Lambda-1, Lambda-2, Gwet’s AC1, and Gwet’s AC2 under Kappa-problematic conditions, with limited AC-2 determinations.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/examining-inter-rater-reliability-of-evaluators-judging-teacher-performance-proposing-an-alternative-to-cohens-kappa/582800/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/examining-inter-rater-reliability-of-evaluators-judging-teacher-performance-proposing-an-alternative-to-cohens-kappa/582800.png","ImageObject",300,407,{"name":92,"@type":93},"Rhys","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-09","2026-10-15",true,{"@type":102,"interactionType":103,"userInteractionCount":39},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"Why is Cohen’s Kappa questioned in evaluating teacher performance ratings?","Question",{"text":112,"@type":113},"Kappa’s validity is challenged when specific rating categories are rare and when raters show high agreement, conditions where Kappa may not adequately correct for chance agreement.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What alternative does the study propose to Cohen’s Kappa?",{"text":117,"@type":113},"The study proposes the Lambda Coefficient of Rater-Mediated Agreement to address concerns about chance correction in rater-mediated assessment processes.",{"name":119,"@type":110,"acceptedAnswer":120},"How do Lambda and other alternative coefficients perform under Kappa-problematic conditions?",{"text":121,"@type":113},"Teacher-profile ratings and simulation results indicate that Lambda-1, Lambda-2, Gwet’s AC1, and Gwet’s AC2 show less variability across raters than Kappa, demonstrating robustness under the data conditions that are problematic for Kappa.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},582800,1792191289,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":39,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":145,"preview_page_count":4},687207024643,"https://ap-avatar.wpscdn.com/davatar_3d24733baf745e90a7e4bdd5f77d97b2","| CEMETR-2021-06 JUNE 2021 | CEME\u003Cbr>Technical Report |\n| --- | --- |\n|  |  |\n| The Center for | Educational Measurement and Evaluation |\n| A PUBLICATION OF THE CENTER FOR EDUCATIONAL MEASUREMENT AND EVALUATION | Examining Inter-Rater Reliability of Evaluators Judging Teacher Performance:\u003Cbr>Proposing an Alternative to Cohen’s Kappa\u003Cbr>Richard G. Lambert\u003Cbr>T. Scott Holcomb\u003Cbr>Bryndle L. Bottoms\u003Cbr>|\n\nExamining Inter-Rater Reliability of Evaluators Judging Teacher Performance: Proposing an Alternative to Cohen’s Kappa  \nRichard Lambert  \nScott Holcomb  \nBryndle Bottoms  \nCenter for Educational Measurement and Evaluation  \nUNC Charlotte  \nPaper presented to the virtual Annual Meeting of the National Council on Measurement in Education, June, 2021.  \nAbstract  \nThe validity of the Kappa coefficient of chance-corrected agreement has been questioned when the prevalence of specific rating scale categories is low and agreement between raters is high. The researchers proposed the Lambda Coefficient ofRater-Mediated Agreement as an alternative to Kappa to address these concerns. Lambda corrects for chance agreement based on specific assumptions about raters and the rater-mediated assessment process including raterspecific tendencies for strict or lenient ratings. Actual ratings of teacher profiles from an interrater reliability exercise confirmed the shortcomings of Kappa. The rater data also demonstrated the robustness of Lambda-1, Lambda-2, Gwet’s AC1, and Gwet’s AC2 to the data conditions that are problematic for Kappa. All four alternative chance-corrected agreement coefficients showed less variability across the 45 raters than Kappa. However, AC-2 was undetermined for 39 of the 45 raters. Simulation data demonstrated the robustness of the Lambda Coefficient of Rater-Mediated Agreement to the data conditions that are problematic for Kappa.  \nExamining Inter-Rater Reliability of Evaluators Judging Teacher Performance:  \nAn Alternative to Cohen’s Kappa  \nCohen’s Kappa (Cohen, 1960) and Weighted Kappa (Cohen, 1968) are widely used measures of chance-corrected agreement between raters. Various questions have been raised about whether Kappa is actually correcting for chance agreement, whether it is useful for identifying and separating various sources of disagreement, the validity of Kappa coefficients when prevalence of specific categories on a rating scale is low, the validity of Kappa coefficients when agreement is high, and the generalizability of Kappa coefficients across populations and study conditions (Thompson & Walter, 1988; Feinstein & Cicchetti; 1990; Cicchetti & Feinstein, 1990) . Gwet (2008) introduced AC1 and AC2 as alternatives to Kappa and demonstrated they are robust indexes not susceptible to the identified shortcomings of Kappa.  \nThis study proposes an alternative to Kappa, Weighted Kappa, AC1, and AC2 that is rooted in theory regarding rated-mediated assessment (Engelhard & Wind, 2018) . The Lambda Coefficient ofRater-Mediated Agreement is designed for use with ordinal scales that are often used to evaluate teacher performance. It examines inter-rater agreement corrected for the probability that raters may agree with expert raters by chance due to the response process they employ when they are uncertain about how to place a teacher on a rubric.  \nRater-Mediated Assessment Theory  \nMost research on rater cognition focuses on the mental processes used by raters of student or examinee performances. The rating process followed by the raters judging constructed responses is intricate, however the judgment of teacher performance can pose even greater complexities. Similar to other examples ofrater-mediated assessments, an observers’ level of expertise, and the overall scoring task demands, drive ratings of teachers (Bell et al., 2018; Suto,  \n2012). Understanding rater cognition is crucial to making a validity argument to support the use of any rater-mediated assessment measure. According to Standard 1.12, which address","cbCaiclFM0XsQA3y","https://ap.wps.com/l/cbCaiclFM0XsQA3y","pdf",707426,33,"English","# Abstract\n# Background: Cohen’s Kappa and Weighted Kappa\n# Alternative Framework: Rater-Mediated Assessment Theory","[{\"question\":\"Why is Cohen’s Kappa questioned in evaluating teacher performance ratings?\",\"answer\":\"Kappa’s validity is challenged when specific rating categories are rare and when raters show high agreement, conditions where Kappa may not adequately correct for chance agreement.\"},{\"question\":\"What alternative does the study propose to Cohen’s Kappa?\",\"answer\":\"The study proposes the Lambda Coefficient of Rater-Mediated Agreement to address concerns about chance correction in rater-mediated assessment processes.\"},{\"question\":\"How do Lambda and other alternative coefficients perform under Kappa-problematic conditions?\",\"answer\":\"Teacher-profile ratings and simulation results indicate that Lambda-1, Lambda-2, Gwet’s AC1, and Gwet’s AC2 show less variability across raters than Kappa, demonstrating robustness under the data conditions that are problematic for Kappa.\"}]","Examining Inter-Rater Reliability of Evaluators Judging Teacher Performance - Proposing an Alternative to Cohen’s Kappa | PDF",1792044525,83]