[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-620977-105":59,"doc-detail-620977-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","judging-the-efficacy-of-artificial-exemplar-material-study-on-standardisation-for-examiner-training","Judging the efficacy of artificial exemplar material - Study on standardisation for examiner training","","Before examiners mark England’s national examinations, they must complete standardisation training to use the mark scheme consistently. This study tests whether generating artificial exemplar material in advance can improve a time-critical process that is otherwise slow and may not reveal a full range of responses. Four exemplar conditions are compared, including artificial pre- and post-examination, standardisation exemplars, and pseudo-random live exemplars, using GCSE Geography questions.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":20,"@type":76,"position":81},"https://docshare.wps.com/document/exam/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/judging-the-efficacy-of-artificial-exemplar-material-study-on-standardisation-for-examiner-training/620977/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/judging-the-efficacy-of-artificial-exemplar-material-study-on-standardisation-for-examiner-training/620977.png","ImageObject",300,407,{"name":92,"@type":93},"Quinn Holloway","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-01","2026-10-20",true,{"@type":102,"interactionType":103,"userInteractionCount":24},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What problem does the study address in examiner standardisation?","Question",{"text":112,"@type":113},"Standardisation requires selecting exemplar scripts from live answers, which is time-consuming and may not uncover a full range of candidate responses during a period of extreme time pressure.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does the study create artificial exemplars?",{"text":117,"@type":113},"The Principal Examiner writes artificial exemplar responses either before the examination is sat (artificial-pre) or after it is sat but before live marking (artificial-post), ensuring coverage of available marks for each question.",{"name":119,"@type":110,"acceptedAnswer":120},"What does the study conclude about perceived exemplar quality across conditions?",{"text":121,"@type":113},"There is no apparent difference in the perceived quality of response dependent on the exemplar condition, though randomly selected live exemplars may be harder to judge. The study recommends further research on marking reliability.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},620977,1792482442,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":19,"category_name":20,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":24,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":145,"preview_page_count":4},2336474466712,"https://ap-avatar.wpscdn.com/davatar_a8503ba1806abce46bf441b54a3ca4cd","Centre for Education Research and Practice  \nJudging the efficacy of artificial exemplar material  \nMithra Vijayathasan, Anne Pinot de Moira and Neil Stringer  \nSummary  \nBefore any examiner can mark national examinations in England they must be trained to use the mark scheme in a process called standardisation. The selection of exemplar material from live scripts for use in standardisation is time consuming and often fails to unearth a full range of candidate responses. This study investigates the possibility of improving the process by generating artificial exemplars in advance of the time-critical period. As a first step, it investigates whether there is any detectable difference in the quality of an exemplar dependent upon how it is created. Four different conditions are included: artificial exemplars written by the Principal Examiner in advance of the examination being sat; artificial exemplars written by the Principal Examiner after the examination is sat; standardisation exemplars selected in the traditional manner; and randomly selected live exemplars.  \nThe study concludes that there is no apparent difference in the perceived quality of response dependent upon condition. However, exemplars randomly selected from the live scripts may be harder to judge. It is recommended that future research should focus on whether artificial exemplars can give rise to comparable, or higher, levels of marking reliability.  \nKeywords: comparative judgement, standardisation, examiner training, exemplars  \nIntroduction  \nThe time between the date of an examination and the awarding of grades is known as the marking period. During this period, responses are received from centres, examiners are trained (or standardised) in the use of the mark scheme, and the marking is undertaken. This must all be completed in time for the grade boundaries to be determined in an award meeting, and for the data from each candidate to be processed. Each stage of the marking period is subject to extreme time pressure. Improvements are continually sought to improve the process.  \nThe standardisation of examiners has undergone many changes in the past few years. On the whole, it is no longer a face-to-face training exercise. It is delivered online and is a prerequisite for marking. Part of the process involves exemplification of aspects of the mark scheme by use of live responses. These live responses – or exemplars, as they are called – are augmented by commentaries. The selection of exemplars is a time-consuming process. It takes place in the first few days after the examination is sat and must be completed as soon as possible to maximise the time available for the marking itself.  \nThis study considers the proposal that creating artificial exemplars in advance of the date of the examination might allow online standardisation to take place earlier; thus increasing the time for marking, while exemplifying a fuller range of candidate responses. Artificial exemplars would be responses written by the Principal Examiner to illustrate given qualities within an answer and to help with application of the mark scheme. In theory, if artificial exemplars could be shown toelicit similar marking behaviours and levels of marking reliability, they could be used to replace the current time-consuming and time-constrained script selection process. As a first step, the study investigates whether the artificial exemplars are distinguishable from exemplars selected  \n[cerp.org.uk](cerp.org.uk)  \nCopyright © 2016 AQA Education and its licensors. All rights reserved.  \nAQA Education (AQA) is a registered charity (registered charity number 1073334) and a company limited by guarantee registered in England and Wales (company number 3644723) . Registered address: AQA, Devas Street, Manchester M15 6EX.  \nfrom live scripts. In other words, whether there are any systematic differences between artificial and live exemplars judged to be of the same quality by the Principal Examiner.  \nData  \n","cbCaibWGvuWTOhYl","https://ap.wps.com/l/cbCaibWGvuWTOhYl","pdf",869334,13,"English","# Summary\n# Introduction\n# Data\n## Study design\n## The four conditions","[{\"question\":\"What problem does the study address in examiner standardisation?\",\"answer\":\"Standardisation requires selecting exemplar scripts from live answers, which is time-consuming and may not uncover a full range of candidate responses during a period of extreme time pressure.\"},{\"question\":\"How does the study create artificial exemplars?\",\"answer\":\"The Principal Examiner writes artificial exemplar responses either before the examination is sat (artificial-pre) or after it is sat but before live marking (artificial-post), ensuring coverage of available marks for each question.\"},{\"question\":\"What does the study conclude about perceived exemplar quality across conditions?\",\"answer\":\"There is no apparent difference in the perceived quality of response dependent on the exemplar condition, though randomly selected live exemplars may be harder to judge. The study recommends further research on marking reliability.\"}]","Judging the efficacy of artificial exemplar material - Study on standardisation for examiner training | PDF",1792464819,33]