[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-620862-105":3,"detail-sidebar-cat-0-en-105":80,"doc-detail-620862-en":130},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":73,"head_meta":75,"extra_data":77,"updated_unix":79},105,"en","online-standardisation-trial-winter-2008-evaluation-of-examiner-performance-and-examiner-satisfaction","Online standardisation trial - Winter 2008 - Evaluation of examiner performance and examiner satisfaction","","January 2008 delivered a second trial of online standardisation across six components involving 74 examiners, with half marked conventionally and half at item level and onscreen using CMI+. For conventionally marked components, no significant differences emerged between paper-based second-phase samples completed in January 2007 and January 2008. Larger differences appeared for online first and second-phase samples, consistent with procedural effects from sample selection and completion. Data limitations affected CMI+ item-marking conclusions, while questionnaires indicated ongoing concerns, partly linked to technical difficulties during the second trial.",{"@graph":14,"@context":72},[15,34,55],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/exam/","Exam",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/online-standardisation-trial-winter-2008-evaluation-of-examiner-performance-and-examiner-satisfaction/620862/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/online-standardisation-trial-winter-2008-evaluation-of-examiner-performance-and-examiner-satisfaction/620862.png","ImageObject",300,407,{"name":42,"@type":43},"Riley West","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-10-21","2026-10-20",true,{"@type":52,"interactionType":53,"userInteractionCount":22},"InteractionCounter",{"@type":54},"ViewAction",{"@type":56,"mainEntity":57},"FAQPage",[58,64,68],{"name":59,"@type":60,"acceptedAnswer":61},"What was the main goal of the January 2008 online standardisation trial?","Question",{"text":62,"@type":63},"To evaluate how online standardisation affected examiner marking, using comparisons across multiple components and marking conditions.","Answer",{"name":65,"@type":60,"acceptedAnswer":66},"How did conventionally marked components perform between January 2007 and January 2008?",{"text":67,"@type":63},"For two of three conventionally marked components, there was no significant difference in absolute mark differences between the paper-based second-phase samples completed in January 2007 and January 2008.",{"name":69,"@type":60,"acceptedAnswer":70},"What concerns did examiners report about the transition to online standardisation?",{"text":71,"@type":63},"Evaluation questionnaire findings indicated that examiners still had concerns, with some linked to technical difficulties experienced with the online resource during the second trial.","https://schema.org",{"og:url":32,"og:type":74,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":76,"canonical":32},"index,follow",{"doc_id":78,"site_id":7},620862,1792568482,{"code":4,"msg":81,"data":82},"success",[83,87,91,94,99,104,109,114,119,122,126],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":84,"show_sort_weight":85,"slug":86},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":88,"show_sort_weight":89,"slug":90},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":92,"slug":93},70,"exam",{"id":95,"doc_module":4,"doc_module_name":25,"category_name":96,"show_sort_weight":97,"slug":98},5,"Comic",60,"comic",{"id":100,"doc_module":4,"doc_module_name":25,"category_name":101,"show_sort_weight":102,"slug":103},6,"Technology",50,"technology",{"id":105,"doc_module":4,"doc_module_name":25,"category_name":106,"show_sort_weight":107,"slug":108},7,"Healthcare",40,"healthcare",{"id":110,"doc_module":4,"doc_module_name":25,"category_name":111,"show_sort_weight":112,"slug":113},8,"Research & Report",30,"research-report",{"id":115,"doc_module":4,"doc_module_name":25,"category_name":116,"show_sort_weight":117,"slug":118},9,"Religion & Spirituality",20,"religion-spirituality",{"id":117,"doc_module":4,"doc_module_name":25,"category_name":120,"show_sort_weight":117,"slug":121},"World Cup","world-cup",{"id":123,"doc_module":4,"doc_module_name":25,"category_name":124,"show_sort_weight":123,"slug":125},10,"Lifestyle","lifestyle",{"id":127,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":95,"slug":129},19,"General","general",{"code":4,"msg":81,"data":131},{"doc_id":78,"user_id":132,"nickname":42,"user_avatar":133,"doc_module":4,"category_id":33,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":22,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":139,"language":140,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":12,"update_tm":144,"read_time":145,"preview_page_count":4},1099523885074,"https://ap-avatar.wpscdn.com/davatar_9964176cb1d06d4a9deccf72a44ae3dc","# Online standardisation trial,Winter 2008:Evaluation of examinerperformance and examiner satisfaction\n\nLucy Billington &Chloe Davenport  \n## SUMMARY\n\nIn January 2008,online standardisation was trialled for the second time across six components.Half of the components were marked conventionally and half at item level and onscreen usingCMl+.For the conventionally marked components,examiners'quality of marking was exploredusing first and second phase sample data from the January 2007 and January 2008 series.Fortwo out of three conventionally marked components,there was no significant difference in termsof absolute mark differences between the paper-based second phase samples completed inJanuary 2007(following face-to-face standardisation)and January 2008(following onlinestandardisation).Larger absolute mark differences were observed for online first and secondphase samples(following online standardisation)than for paper-based samples(followingface-to-face standardisation).This finding is consistent with those from the first trial in June2007,and is believed to be a result of procedural differences in the way that the two types ofsamples are selected and completed.  \nThe GCE Sport and Physical Education component,PED4 was included in the first onlinestandardisation trial in June 2007.The findings for this component were exceptional,in thatmode of standardisation appeared to have impacted upon the accuracy of the examiners'second phase sample paper-based marking.In addition,their online second phase samplemarking was found to have deteriorated since the previous trial.The mark scheme for theJanuary 2008 paper may have been more complex and translated less well to onlinestandardisation.Also an increased workload,due to participation in a research exercise andbeing standardised online for an additional component(PED1 was introduced to onlinestandardisation in the second trial),offer potential explanations for the examiners'reducedmarking reliability.  \nLimited data was available for the three components marked by CMl+but it appeared thatonline standardisation did not impact upon the examiners'ability to mark seed items.Thefindings for the January 2008 data are discussed in the light of the recent findings of otheronline standardisation research initiatives e.g.an experimental study conducted with GCSEHistory examiners.Whilst there is substantial quantitative evidence to support the extension ofonline standardisation to other components,qualitative findings from evaluation questionnairesadministered to participants suggest that examiners still have concerns regarding the transitionfrom face-to-face to online standardisation.  Some of these concerns stem from technicaldifficulties experienced with the online resource during the second trial.  \nwww.cerp.org.uk  \nCopyright G2012 AQA and its licensors.All rights reserved.  \nThe Assessment and Qualfications Aliance(AQA)is a company imited by guarantee registered in England and Wales(company number3644723).Registered address:AQA,Devas Street,Manchester M156EX  \n## BACKGROUND\n\nIn January 2008 the Assessment and Qualifications Alliance(AQA)trialled its onlinestandardisation facility for the second time.The trial included a total of six components (seeTable 1)and involved 74 examiners.Only two components(3820/3 and PED4)were includedin the previous online standardisation trial,which took place in June 2007.January 2008 wasthe first time that online standardisation had been combined with CMl+.CMl+is a softwarepackage that enables examiners to mark scanned images of scripts,using their home PC.Examiners mark scripts directly on-screen at item level.  \nTable 1:Components included in the January 2008 online standardisation trial.  \n\n| Qualification   | Code   | Title   | CMl+  \u003Cbr>Jan  \u003Cbr>08   |\n| --- | --- | --- | --- |\n| GCSE Health &Social Care   | *3820/3   | Understanding Personal Development and  \u003Cbr>Relationships   | x   |\n| GCE Sport&Physical  \u003Cbr>Education   | PED1   | Physiological and ","cbCaibyeSfqDkKYk","https://ap.wps.com/l/cbCaibyeSfqDkKYk","pdf",547374,21,"English","# SUMMARY\n## Marking environments and sample comparisons\n## CMI+ item-level marking findings\n## Examiner questionnaire concerns\n# BACKGROUND\n## Trial scope and components\n## CMI+ marking process\n## Prior June 2007 online trial findings","[{\"question\":\"What was the main goal of the January 2008 online standardisation trial?\",\"answer\":\"To evaluate how online standardisation affected examiner marking, using comparisons across multiple components and marking conditions.\"},{\"question\":\"How did conventionally marked components perform between January 2007 and January 2008?\",\"answer\":\"For two of three conventionally marked components, there was no significant difference in absolute mark differences between the paper-based second-phase samples completed in January 2007 and January 2008.\"},{\"question\":\"What concerns did examiners report about the transition to online standardisation?\",\"answer\":\"Evaluation questionnaire findings indicated that examiners still had concerns, with some linked to technical difficulties experienced with the online resource during the second trial.\"}]","Online standardisation trial - Winter 2008 - Evaluation of examiner performance and examiner satisfaction | PDF",1792464694,53]