[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-590556-105":3,"detail-sidebar-cat-0-en-105":80,"doc-detail-590556-en":130},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":73,"head_meta":75,"extra_data":77,"updated_unix":79},105,"en","measuring-program-quality-evidence-of-the-scientific-validity-of-the-assessment-of-program-practices-tool","Measuring Program Quality - Evidence of the Scientific Validity of the Assessment of Program Practices Tool","","Out-of-school time (OST) youth programs are difficult to assess because they are dynamic and occur across varied settings with frequent staff–youth interactions. Measuring program quality is presented as essential for guiding directors and policymakers on where to improve and how to support improvement. The article reports research validating the Assessment of Program Practices Tool (APT), including two validation phases that examine scientific rigor, support tool improvements, and inform training to strengthen rating reliability for high-stakes users.",{"@graph":14,"@context":72},[15,34,55],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/measuring-program-quality-evidence-of-the-scientific-validity-of-the-assessment-of-program-practices-tool/590556/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/measuring-program-quality-evidence-of-the-scientific-validity-of-the-assessment-of-program-practices-tool/590556.png","ImageObject",300,407,{"name":42,"@type":43},"Stanley","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-05","2026-10-15",true,{"@type":52,"interactionType":53,"userInteractionCount":33},"InteractionCounter",{"@type":54},"ViewAction",{"@type":56,"mainEntity":57},"FAQPage",[58,64,68],{"name":59,"@type":60,"acceptedAnswer":61},"Why is measuring quality in out-of-school time (OST) programs challenging?","Question",{"text":62,"@type":63},"OST programs are inherently dynamic, with many youth interacting with each other and staff across different physical environments, making assessment harder than in more controlled settings.","Answer",{"name":65,"@type":60,"acceptedAnswer":66},"What is the Assessment of Program Practices Tool (APT) used for?",{"text":67,"@type":63},"The APT measures OST process quality by capturing observable aspects of programs in action and supports program self-assessment, improvement, and external quality rating.",{"name":69,"@type":60,"acceptedAnswer":70},"What does Phase 1 of the APT Validation Study evaluate?",{"text":71,"@type":63},"Phase 1 evaluates the tool’s technical properties and scientific rigor using tests such as internal consistency, test-retest stability, interrater reliability, and predictive validity.","https://schema.org",{"og:url":32,"og:type":74,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":76,"canonical":32},"index,follow",{"doc_id":78,"site_id":7},590556,1792189578,{"code":4,"msg":81,"data":82},"success",[83,87,91,95,100,105,110,114,119,122,126],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":84,"show_sort_weight":85,"slug":86},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":88,"show_sort_weight":89,"slug":90},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":92,"show_sort_weight":93,"slug":94},"Exam",70,"exam",{"id":96,"doc_module":4,"doc_module_name":25,"category_name":97,"show_sort_weight":98,"slug":99},5,"Comic",60,"comic",{"id":101,"doc_module":4,"doc_module_name":25,"category_name":102,"show_sort_weight":103,"slug":104},6,"Technology",50,"technology",{"id":106,"doc_module":4,"doc_module_name":25,"category_name":107,"show_sort_weight":108,"slug":109},7,"Healthcare",40,"healthcare",{"id":111,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":112,"slug":113},8,30,"research-report",{"id":115,"doc_module":4,"doc_module_name":25,"category_name":116,"show_sort_weight":117,"slug":118},9,"Religion & Spirituality",20,"religion-spirituality",{"id":117,"doc_module":4,"doc_module_name":25,"category_name":120,"show_sort_weight":117,"slug":121},"World Cup","world-cup",{"id":123,"doc_module":4,"doc_module_name":25,"category_name":124,"show_sort_weight":123,"slug":125},10,"Lifestyle","lifestyle",{"id":127,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":96,"slug":129},19,"General","general",{"code":4,"msg":81,"data":131},{"doc_id":78,"user_id":132,"nickname":42,"user_avatar":133,"doc_module":4,"category_id":111,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":33,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":115,"language":139,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":12,"update_tm":143,"read_time":144,"preview_page_count":4},2336477405376,"https://ap-avatar.wpscdn.com/davatar_29158cc5080c5b710cf443261637dec0","NEW FROM NIOST  \nMeasuring Program Quality:  \nEvidence of the Scientific Validity of the Assessment of Program Practices Tool  \nAllison Tracy, Linda Charmaraman, Ineke Ceder, Amanda Richer, and Wendy Surr  \nOut-of-school time (OST) youth programs are inherently difficult to assess. They are often very dynamic: Many youth interact with one another and with staff members in various physical environments. Despite the challenge, measuring quality is critical to help program directors and policy makers identify where to improve and how to support those improvements.  \nThis article describes recent research on the Assessment of Program Practices Tool (APT), establishing its strength as an evaluation and tracking tool for OST programs. Funded by the William T. Grant Foundation and Virginia B. Toulmin Foundation, the validation was conducted in two phases. The first phase was designed to evaluate the scientific rigor of the tool. Based on the findings from the first phase, the second aimed to inform improvements in the tool and its training. Our testing so far shows that online video-based training needs to be more specialized in order to improve rating reliability for high-stakes users, such as third-party evaluators.  \nALLISON TRACY has over 15 years of experience providing methodological and statistical consultation for researchers in a wide variety of disciplines, topics, and institutions. She has worked on the psychometric analysis of A Program Assessment System (APAS) and its components, including the Assessment of Program Practices Tool (APT) .  \nLINDA CHARMARAMAN, PhD, co-principal investigator of the APT Validation Study II, is a research scientist at the Wellesley Centers for Women specializing in positive youth development. She has conducted research and evaluations of in-school and out-of-school time (OST) programs for over 12 years.  \nINEKE CEDER is a research associate at the Wellesley Centers for Women, where she has been involved with projects on child and adolescent development, sexual education, and women’s leadership. She was the data collection manager and survey developer for Phase 2 of the APT Validation Study.  \nAMANDA RICHER is a research associate for the National Institute on Out-of-School Time and assistant methodologist for the Wellesley Centers for Women. She has been involved in psychometric testing of OST assessments and she has supported research in youth development. WENDY SURR, Senior Researcher at the American Institutes for Research, has more than 30 years of experience leading research and evaluation studies and other educational initiatives. While at NIOST, she co-created APAS, including developing the APT and a set of teacher, staff, and youth survey instruments for measuring student nonacademic outcomes. She served as co-principal investigator and project director for Phase 1 of the APT Validation Study.  \nTracy, Charmaraman, Ceder, Richer, & Surr MEASURING PROGRAM QUALITY 3  \nThe Assessment of Program Practices Tool  \nThe APT is an observational instrument created in 2005 by Beth Miller and Wendy Surr of the National Institute on Out-of-School Time (NIOST) to measure OST process quality: observable aspects of a program in action. Research suggests that process quality contributes to the 21st century skills, attitudes, and behaviors youth need tobe successful in school and the workplace (Miller, 2005) .  \nThe APT is one component of A Program Assessment System (APAS), an integrated quality and outcome assessment system developed by NIOST in partnership with the Massachusetts Department of Elementary and Secondary Education 21st Century Community Learning Centers (21st CCLC) initiative. Currently, the APT is used in 33 states and in Canada by over 1,500 individuals and 600 OST programs. The APT is designed to support program self-assessment and improvement efforts. Increasingly, it is also being used by external stakeholders, such as funders and sponsors of afterschool programs, to ensure that programs","cbCaijIS09qzy58e","https://ap.wps.com/l/cbCaijIS09qzy58e","pdf",185587,"English","# Measuring Program Quality\n## Why Program Quality Matters in OST Programs\n## Assessment of Program Practices Tool (APT)\n## APT Domains and Quality Areas\n## Rating Scale, Anchors, and Conditional Items\n## APT Validation Study Phase 1: Scientific Rigor\n## Methods and Participating Programs","[{\"question\":\"Why is measuring quality in out-of-school time (OST) programs challenging?\",\"answer\":\"OST programs are inherently dynamic, with many youth interacting with each other and staff across different physical environments, making assessment harder than in more controlled settings.\"},{\"question\":\"What is the Assessment of Program Practices Tool (APT) used for?\",\"answer\":\"The APT measures OST process quality by capturing observable aspects of programs in action and supports program self-assessment, improvement, and external quality rating.\"},{\"question\":\"What does Phase 1 of the APT Validation Study evaluate?\",\"answer\":\"Phase 1 evaluates the tool’s technical properties and scientific rigor using tests such as internal consistency, test-retest stability, interrater reliability, and predictive validity.\"}]","Measuring Program Quality - Evidence of the Scientific Validity of the Assessment of Program Practices Tool | PDF",1792069876,23]