[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-621026-105":59,"doc-detail-621026-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","examiner-background-and-the-effect-on-marking-reliability-march-2003","Examiner Background and the Effect on Marking Reliability - March 2003","","Examiner Background and the Effect on Marking Reliability presents a study of how examiner backgrounds relate to marking reliability in GCE A Level assessment. Using data from 21 AQA A2 units marked by 356 examiners in summer 2002, four multilevel models test several statistical measures of agreement between senior and assistant examiners, including adjustment probability and examiner performance ratings. Results find no clear link to personal characteristics, while reliability appears connected to allocation features and subject-specific idiosyncrasies, with related centre-type differences explored via principal examiner questionnaires.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/examiner-background-and-the-effect-on-marking-reliability-march-2003/621026/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/examiner-background-and-the-effect-on-marking-reliability-march-2003/621026.png","ImageObject",300,407,{"name":92,"@type":93},"Angel","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-28","2026-10-20",true,{"@type":102,"interactionType":103,"userInteractionCount":19},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What data and marking setup does the study use?","Question",{"text":112,"@type":113},"The study analyzes marks from 21 AQA A2 units, assessed by 356 examiners in summer 2002. It compares senior and assistant examiner marking and models reliability-related outcomes at multiple levels.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"Does the report find a connection between examiner personal characteristics and marking reliability?",{"text":117,"@type":113},"No. The study identifies no link between personal characteristics and marking reliability, suggesting that reliability is driven more by examiner allocation features and subject-specific idiosyncrasies.",{"name":119,"@type":110,"acceptedAnswer":120},"Which models and measures are used to assess marking reliability?",{"text":121,"@type":113},"Four multilevel models are fitted using different measures: the difference between senior and assistant examiner marks, the absolute difference, the probability of a numerical adjustment to assistant marks, and an examiner performance rating measure.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},621026,1792546996,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":19,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":145},687207412472,"https://ap-avatar.wpscdn.com/davatar_155a257f0dc6eb9ab79c44ca47cae57d","Centre for Education Research and Practice  \nExaminer Background and the Effect on Marking Reliability  \nAnne Pinot de Moira Assessment & Qualifications Alliance  \nMay 2003  \nThis research has been funded by QCA following recommendations made in the independent panel report on maintaining GCE A Level standards published in January 2002  \n[cerp.org.uk](cerp.org.uk)  \nCopyright © 2014 AQA Education and its licensors. All rights reserved.  \nAQA Education (AQA) is a registered charity (registered charity number 1073334) and a company limited by guarantee registered in England and Wales (company number 3644723) . Registered address: AQA, Devas Street, Manchester M15 6EX.  \nCentre for Education Research and Practice  \nCONTENTS  \nSUMMARY.............................................................................................................................................. 3  \nINTRODUCTION..................................................................................................................................... 5  \nBACKGROUND........................................................................................................................................ 5  \nAWARDING BODY PRACTICES ................................................................................................................. 5  \nMARK-REMARK RELIABILITY ................................................................................................................... 6  \nBIAS AND CONTEXT EFFECTS ................................................................................................................. 6  \nEXAMINER CHARACTERISTICS ................................................................................................................ 7  \nMETHODOLOGY.................................................................................................................................... 9  \nTHE DATA ............................................................................................................................................. 9  \nTHE MODELS ....................................................................................................................................... 10  \nVARIABLE FORMULATION...................................................................................................................... 11  \nFINDINGS ............................................................................................................................................. 12  \nMODEL 1 – THE PERCENTAGE DIFFERENCE MODEL .............................................................................. 12  \nMODEL 2 – THE ABSOLUTE PERCENTAGE DIFFERENCE MODEL ............................................................. 14  \nMODEL 3 – THE ADJUSTMENT MODEL .................................................................................................. 17  \nMODEL 4 – THE EXAMINER PERFORMANCE RATING MODEL................................................................... 18  \nQUESTIONNAIRE TO PRINCIPAL EXAMINERS .......................................................................................... 20  \nCONCLUSIONS.................................................................................................................................... 24  \nRECOMMENDATIONS......................................................................................................................... 25  \nREFERENCES...................................................................................................................................... 26  \nAPPENDIX A Variable Formulation.................................................................................................... 29  \nAPPENDIX B Residual Plots & Model Diagnostics ............................................................................ 33  \nAPPENDIX C Questionnaire to Principal Examiners ......................................................................... 37  \n1  \nCentre for Educat","cbCaigZSQbSAK33H","https://ap.wps.com/l/cbCaigZSQbSAK33H","pdf",807591,39,"English","# Summary\n# Introduction\n## Background\n## Awarding Body Practices\n## Mark-Remark Reliability\n## Bias and Context Effects\n## Examiner Characteristics\n# Methodology\n## The Data\n## The Models\n## Variable Formulation\n# Findings\n## Model 1 – The Percentage Difference Model\n## Model 2 – The Absolute Percentage Difference Model\n## Model 3 – The Adjustment Model\n## Model 4 – The Examiner Performance Rating Model\n## Questionnaire to Principal Examiners\n# Conclusions\n# Recommendations\n# References\n# Appendix A Variable Formulation\n# Appendix B Residual Plots & Model Diagnostics\n# Appendix C Questionnaire to Principal Examiners","[{\"question\":\"What data and marking setup does the study use?\",\"answer\":\"The study analyzes marks from 21 AQA A2 units, assessed by 356 examiners in summer 2002. It compares senior and assistant examiner marking and models reliability-related outcomes at multiple levels.\"},{\"question\":\"Does the report find a connection between examiner personal characteristics and marking reliability?\",\"answer\":\"No. The study identifies no link between personal characteristics and marking reliability, suggesting that reliability is driven more by examiner allocation features and subject-specific idiosyncrasies.\"},{\"question\":\"Which models and measures are used to assess marking reliability?\",\"answer\":\"Four multilevel models are fitted using different measures: the difference between senior and assistant examiner marks, the absolute difference, the probability of a numerical adjustment to assistant marks, and an examiner performance rating measure.\"}]","Examiner Background and the Effect on Marking Reliability - March 2003 | PDF",1792464874,98]