[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-621032-105":3,"detail-sidebar-cat-0-en-105":80,"doc-detail-621032-en":130},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":73,"head_meta":75,"extra_data":77,"updated_unix":79},105,"en","seeds-of-doubt-learning-lessons-from-item-level-marking","Seeds of Doubt - Learning lessons from item level marking","","This paper proposes a multilevel model to compare marking reliability at both the seed-marked item level and the seed-marked unit level. Using seed data from the CMI+ system, it treats the absolute difference between the seed mark and the examiner mark as a proxy for reliability, with smaller differences indicating more reliable marking. The analysis identifies where unexplained variation occurs and discusses implications for examiner training and the design of assessments and mark schemes.",{"@graph":14,"@context":72},[15,34,55],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/seeds-of-doubt-learning-lessons-from-item-level-marking/621032/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/seeds-of-doubt-learning-lessons-from-item-level-marking/621032.png","ImageObject",300,407,{"name":42,"@type":43},"Angel","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-01","2026-10-20",true,{"@type":52,"interactionType":53,"userInteractionCount":33},"InteractionCounter",{"@type":54},"ViewAction",{"@type":56,"mainEntity":57},"FAQPage",[58,64,68],{"name":59,"@type":60,"acceptedAnswer":61},"What does the paper aim to improve about marking reliability?","Question",{"text":62,"@type":63},"It aims to provide a mechanism to focus efforts on where changes to examiner training and the assessment/mark scheme design can improve marking reliability.","Answer",{"name":65,"@type":60,"acceptedAnswer":66},"How is marking reliability measured in the analysis?",{"text":67,"@type":63},"The paper calculates the absolute difference between the seed mark and the examiner mark for each response, using the seed mark as a proxy for the standard.",{"name":69,"@type":60,"acceptedAnswer":70},"What do the model fit results suggest about where variation comes from?",{"text":71,"@type":63},"Although the model improves over a null model, it explains limited variation overall, with the greatest unexplained variation occurring at the response level.","https://schema.org",{"og:url":32,"og:type":74,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":76,"canonical":32},"index,follow",{"doc_id":78,"site_id":7},621032,1792568326,{"code":4,"msg":81,"data":82},"success",[83,87,91,95,100,105,110,114,119,122,126],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":84,"show_sort_weight":85,"slug":86},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":88,"show_sort_weight":89,"slug":90},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":92,"show_sort_weight":93,"slug":94},"Exam",70,"exam",{"id":96,"doc_module":4,"doc_module_name":25,"category_name":97,"show_sort_weight":98,"slug":99},5,"Comic",60,"comic",{"id":101,"doc_module":4,"doc_module_name":25,"category_name":102,"show_sort_weight":103,"slug":104},6,"Technology",50,"technology",{"id":106,"doc_module":4,"doc_module_name":25,"category_name":107,"show_sort_weight":108,"slug":109},7,"Healthcare",40,"healthcare",{"id":111,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":112,"slug":113},8,30,"research-report",{"id":115,"doc_module":4,"doc_module_name":25,"category_name":116,"show_sort_weight":117,"slug":118},9,"Religion & Spirituality",20,"religion-spirituality",{"id":117,"doc_module":4,"doc_module_name":25,"category_name":120,"show_sort_weight":117,"slug":121},"World Cup","world-cup",{"id":123,"doc_module":4,"doc_module_name":25,"category_name":124,"show_sort_weight":123,"slug":125},10,"Lifestyle","lifestyle",{"id":127,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":96,"slug":129},19,"General","general",{"code":4,"msg":81,"data":131},{"doc_id":78,"user_id":132,"nickname":42,"user_avatar":133,"doc_module":4,"category_id":111,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":33,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":101,"language":139,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":12,"update_tm":143,"read_time":144},687207412472,"https://ap-avatar.wpscdn.com/davatar_155a257f0dc6eb9ab79c44ca47cae57d","Centre for Education Research and Policy  \nSeeds of Doubt  \nLearning lessons from item level marking  \nAnne Pinot de Moira  \nSummary  \nThis paper proposes a multilevel model which could be used to compare marking reliability between seed marked items and between seed marked units. It is hoped that the output from the model might promote the sharing of good practice and allow AQA to present a more unified approach to the design of mark schemes and items.  \nKeywords: marking reliability, seed data, multilevel models  \nIntroduction  \nMore ready access to item level information for units which are marked using the CMI+ system has allowed increased scrutiny of the reliability with which these items are marked. Data are available for each CMI+ unit, each seed within that unit and each occasion on which the seed was remarked; thus allowing comparison of the seed mark and remark to provide a measure of marking reliability.  \nIncreases in reliability can be effected in many ways. It can be argued that mark reliability, in other words reliability of the final mark awarded to a candidate, can be increased by item level marking (see, for example, Wheadon & Pinot de Moira, 2012) or some form of multiple marking (see, for example, Meadows & Billington, 2005; Pilliner, 1968) . In contrast, and here it is important to make a distinction, marking reliability can only be improved by examiner training and improvements to the assessment and mark scheme. This paper provides a mechanism by which to focus efforts on areas where changes to training and the assessment might improve marking reliability. Whilst technical in nature, the analysis in this paper is presented with the introduction of new operational practices in mind.  \nA model of marking reliability  \nThe data  \nIn 2012, there were 5,335,217 seed marked responses recorded during the marking window. The analysis of marking reliability presented here used a stratified sample of these responses. The sampling frame was restricted to include only items with a maximum mark tariff of two or more and to include a maximum of 500 responses per item (525,310 responses met these criteria) . The final sample comprised 52,991 marks awarded to seed items and exemplified the work on 3,351 items from 235 units.  \nFor each response, the absolute difference between the seed mark and the examiner mark was calculated. Because of the examiner hierarchy observed within the system and the fact that the seed mark is assigned by a senior examiner, the seed mark was assumed to represent the gold  \n[www.cerp.org.uk](www.cerp.org.uk)  \nCopyright © 2013 AQA Education and its licensors. All rights reserved.  \nAQA Education (AQA) is a registered charity (number 1073334) and a company limited by guarantee registered in England and Wales (number 3644723) . Our registered address is AQA, Devas Street, Manchester M15 6EX.  \nstandard. Therefore, the difference from the seed mark was taken as a proxy for marking reliability: the smaller the difference, the more reliable the marking.  \nThe multilevel model  \nTo improve marking reliability, it is important to focus on the sources of error. It is relatively straightforward, by use of simple statistical measures, to identify examiners whose marking is at odds with the standard. However, errors are also likely to come from the assessment, the item or the mark scheme design as well as the examiners themselves. To gain a fuller picture of marking reliability requires modelling the interrelations between all these sources.  \nThus, a three-level linear multilevel model was fitted to assess marking reliability for the differing items and units (Equation 1) . Seed marked responses (i) were nested within items (j) and items were nested within units (k) . The absolute difference between the seed mark and examiner mark was used as the dependent variable. The maximum mark for the item was included as an independent variable. The parameter estimates associated with the model are included in Ap","cbCainL33w7MFGba","https://ap.wps.com/l/cbCainL33w7MFGba","pdf",257215,"English","# Summary\n# Introduction\n# A model of marking reliability\n## The data\n## The multilevel model\n## The model fit and limitations\n# Output","[{\"question\":\"What does the paper aim to improve about marking reliability?\",\"answer\":\"It aims to provide a mechanism to focus efforts on where changes to examiner training and the assessment/mark scheme design can improve marking reliability.\"},{\"question\":\"How is marking reliability measured in the analysis?\",\"answer\":\"The paper calculates the absolute difference between the seed mark and the examiner mark for each response, using the seed mark as a proxy for the standard.\"},{\"question\":\"What do the model fit results suggest about where variation comes from?\",\"answer\":\"Although the model improves over a null model, it explains limited variation overall, with the greatest unexplained variation occurring at the response level.\"}]","Seeds of Doubt - Learning lessons from item level marking | PDF",1792464882,15]