[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-620825-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-620825-en":130},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","a-review-of-the-literature-on-marking-reliability-a-study-on-reliability-in-assessment-and-marking","A REVIEW OF THE LITERATURE ON MARKING RELIABILITY - A Study on Reliability in Assessment and Marking","","A review of the literature on marking reliability examines how assessment scores vary due to sources of unreliability in marking, including factors tied to tests, candidates, scoring processes, and examiner characteristics. It introduces classical test theory, generalisability theory, and item response theory, then details definitions and estimation approaches for reliability. The review analyses interrater reliability types, bias sources, effects of question format and mark schemes, and practical measures to detect and correct unreliable marking, including training, double marking, and e-marking.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/a-review-of-the-literature-on-marking-reliability-a-study-on-reliability-in-assessment-and-marking/620825/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/a-review-of-the-literature-on-marking-reliability-a-study-on-reliability-in-assessment-and-marking/620825.png","ImageObject",300,407,{"name":42,"@type":43},"Alex Sinclair","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-10-28","2026-10-20",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",5,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What does the review focus on regarding marking reliability?","Question",{"text":63,"@type":64},"It focuses on marking reliability across externally assessed examination scripts, describing levels of reliability achieved in different assessment forms and research methods for improving reliability.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"Which measurement theories are introduced in the review?",{"text":68,"@type":64},"The review introduces classical test theory, generalisability theory, and item response theory (IRT) and explains how they influence the definition and measurement of assessment reliability.",{"name":70,"@type":61,"acceptedAnswer":71},"What factors does the review discuss as sources of unreliability or bias in marking?",{"text":72,"@type":64},"It discusses sources of unreliability tied to the test, the candidates, scoring factors, and examiner characteristics, and it also covers bias such as contrast effects and gender and ethnic bias.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},620825,1792521243,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,100,105,110,114,119,122,126],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":55,"doc_module":4,"doc_module_name":25,"category_name":97,"show_sort_weight":98,"slug":99},"Comic",60,"comic",{"id":101,"doc_module":4,"doc_module_name":25,"category_name":102,"show_sort_weight":103,"slug":104},6,"Technology",50,"technology",{"id":106,"doc_module":4,"doc_module_name":25,"category_name":107,"show_sort_weight":108,"slug":109},7,"Healthcare",40,"healthcare",{"id":111,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":112,"slug":113},8,30,"research-report",{"id":115,"doc_module":4,"doc_module_name":25,"category_name":116,"show_sort_weight":117,"slug":118},9,"Religion & Spirituality",20,"religion-spirituality",{"id":117,"doc_module":4,"doc_module_name":25,"category_name":120,"show_sort_weight":117,"slug":121},"World Cup","world-cup",{"id":123,"doc_module":4,"doc_module_name":25,"category_name":124,"show_sort_weight":123,"slug":125},10,"Lifestyle","lifestyle",{"id":127,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":55,"slug":129},19,"General","general",{"code":4,"msg":82,"data":131},{"doc_id":79,"user_id":132,"nickname":42,"user_avatar":133,"doc_module":4,"category_id":111,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":139,"language":140,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":12,"update_tm":144,"read_time":145},1099523882182,"https://ap-avatar.wpscdn.com/davatar_6f874abed73319feea01a86fa6f0fab8","A REVIEW OF THE LITERATURE ON MARKING RELIABILITY  \nMichelle Meadows and Lucy Billington  \nMay 2005  \nThis report was commissioned by the National Assessment Agency  \nCONTENTS  \nINTRODUCTION 4  \nDEFINITIONS AND FORMS OF RELIABILITY 4  \nA Brief Introduction to Classical Test Theory, Generalisability Theory and Item Response Theory 4  \nDefinitions of reliability 7  \nSources of unreliability 7  \nThe test 7  \nThe candidates 8  \nScoring factors 8  \nEstimating reliability 9  \nGrades / levels 10  \nMaking tests more reliable 11  \nUsing tests to predict future performance 12  \nUsing tests to select individuals 13  \nThe relationship between reliability and validity 13  \nTypes of interrater reliability 14  \nConsensus estimates of reliability 14  \nConsistency estimates of reliability 14  \nMeasurement estimates of reliability 15  \nThe limitations of the correlation as a measure of reliability 16  \nTHE UBIQUITOUS RELIABILITY OF MARKING STUDY 17  \nCHANGES IN THE CONSISTENCY AND SEVERITY OF MARKING OVER TIME 20  \nSOURCES OF BIAS IN MARKING 23  \nContrast effects 23  \nThe text of the script 25  \nThe candidate 27  \nGender bias 27  \nEthnic bias 30  \nThe examiner 30  \nExaminer background 30  \nExaminer traits 34  \nTransient examiner traits 34  \nTHE EFFECT OF QUESTION FORMAT, SUBJECT AND CHOICE OF ESSAY TOPIC ON MARKING RELIABILITY 35  \nQuestion format 35  \nCandidates’ choice of essay topic 38  \nStudies of the process by which examiners rate essays 38  \nImproving the reliability of essay marking 41  \nTHE EFFECT OF THE MARK SCHEME/RATING SYSTEM ON MARKING RELIABILITY 42  \nPROCEDURAL INFLUENCES ON MARKING RELIABILITY 47  \nConsensus versus hierarchical approaches to achieving marking reliability 47  \nTraining and feedback 50  \nCommunity of practice 53  \nExemplar material 55  \nDouble and multiple marking 56  \nREMEDIAL MEASURES TO DETECT/CORRECT UNRELIABLE MARKING 60  \nAdjustments to marks 60  \nMethods for detecting unreliable examiners used by UK awarding bodies 64  \nEnquiries on results 64  \nCentre grade comparison list 64  \nOffice review 64  \nBorderline reviews 65  \nChecking for clerical errors 66  \nThe reliability of e-marking 67  \nCONCLUSIONS 68  \nThe need to routinely report reliability statistics alongside grades 68  \nAlternatives to marking 70  \nThurstone paired comparison of scripts 70  \nComputer marking 70  \nREFERENCES 72  \nAPPENDIX 1 A BRIEF INTRODUCTION TO THE CORRELATION COEFFICIENT 88  \nINTRODUCTION  \nHartog and Rhodes’ seminal book on marking reliability began “No element in the structure of our national education occupies at the present moment more public attention than our system of examinations” (vii, 1936) . The focus of public attention has not diminished over the intervening 70 years. Moreover, there is an assumption on the part of the public that the marks awarded to candidates in high stakes examinations such as GCSE and GCE are (with only the occasional exception) highly reliable and a true reflection of the candidates’ abilities. Yet there is a long history of research findings to the contrary. As early as 1912 Starch and Elliott reported a study of the reliability of the marks assigned by teachers to English examination scripts. There was wide variation in the marks given to the same script. They replicated their findings in the marking of Mathematics (1913a) and History (1913b) . They expected to find more consistency in the marking in Mathematics than English, but found that the marks varied even more widely. They suggested that this was because some teachers took into account the poor appearance of the script, which others ignored. The teachers also came from schools with varying levels of achievement, which they believed could have affected their grading.  \nMarking reliability is the focus of this literature review, which will cover the levels of marking reliability achieved in different forms of assessment and research into methods of improving marking reliability. It concentrates upon the marking of externally assessed exa","cbCais1WoiCqdyYx","https://ap.wps.com/l/cbCais1WoiCqdyYx","pdf",452335,89,"English","# Introduction\n## Definitions and forms of reliability\n## A Brief Introduction to Classical Test Theory, Generalisability Theory and Item Response Theory\n## The ubiquitous reliability of marking study\n## Changes in consistency and severity of marking over time\n## Sources of bias in marking\n## The effect of question format, subject and choice of essay topic on marking reliability\n## The effect of the mark scheme/rating system on marking reliability\n## Procedural influences on marking reliability\n## Remedial measures to detect/correct unreliable marking\n## Conclusions\n## References","[{\"question\":\"What does the review focus on regarding marking reliability?\",\"answer\":\"It focuses on marking reliability across externally assessed examination scripts, describing levels of reliability achieved in different assessment forms and research methods for improving reliability.\"},{\"question\":\"Which measurement theories are introduced in the review?\",\"answer\":\"The review introduces classical test theory, generalisability theory, and item response theory (IRT) and explains how they influence the definition and measurement of assessment reliability.\"},{\"question\":\"What factors does the review discuss as sources of unreliability or bias in marking?\",\"answer\":\"It discusses sources of unreliability tied to the test, the candidates, scoring factors, and examiner characteristics, and it also covers bias such as contrast effects and gender and ethnic bias.\"}]","A REVIEW OF THE LITERATURE ON MARKING RELIABILITY - A Study on Reliability in Assessment and Marking | PDF",1792464647,224]