[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-590320-105":59,"doc-detail-590320-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","improving-the-design-of-evaluations-that-include-students-teachers-and-schools-an-empirical-investigation-of-key-design-parameters","Improving the Design of Evaluations that Include Students, Teachers, and Schools - An Empirical Investigation of Key Design Parameters","","Designing evaluations to detect the effectiveness of educational interventions on student achievement requires careful planning within a complex multilevel learning environment. This empirical study estimates key statistical design parameters for planning adequately powered cluster randomized trials that include student, teacher, and school levels focused on reading, mathematics, or science. Using administrative datasets from Michigan, North Carolina, Kentucky, and Maryland for grades 3–8, results show variance in student test scores is typically larger between teachers within schools than between schools.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/improving-the-design-of-evaluations-that-include-students-teachers-and-schools-an-empirical-investigation-of-key-design-parameters/590320/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/improving-the-design-of-evaluations-that-include-students-teachers-and-schools-an-empirical-investigation-of-key-design-parameters/590320.png","ImageObject",300,407,{"name":92,"@type":93},"Pentious","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-06","2026-10-15",true,{"@type":102,"interactionType":103,"userInteractionCount":52},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"Why is statistical power analysis important for evaluations in this study?","Question",{"text":112,"@type":113},"Because evaluations must account for the multilevel structure of students nested within teachers and teachers nested within schools to detect intervention effects with sufficient power.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What data sources and grade range were used in the analysis?",{"text":117,"@type":113},"The study uses administrative state datasets from Michigan, North Carolina, Kentucky, and Maryland for grades 3 through 8.",{"name":119,"@type":110,"acceptedAnswer":120},"What does the study find about where student test score variance is largest?",{"text":121,"@type":113},"With few exceptions, student test score variance is larger between teachers within schools than between schools.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},590320,1792166222,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":52,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":46,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":143,"read_time":31,"preview_page_count":4},1374404730887,"https://ap-avatar.wpscdn.com/davatar_6f874abed73319feea01a86fa6f0fab8","AERA Open  \nJanuary-December 2025, Vol. 11, No. 1, pp. 1–20 DOI: 10. 1177/23328584251320380  \nArticle reuse guidelines: [sagepub.com/journals-permissions](sagepub.com/journals-permissions)[ ](sagepub.com/journals-permissions)© The Author(s) 2025. [https://journals.sagepub.com/home/ero](https://journals.sagepub.com/home/ero)  \nImproving the Design of Evaluations that Include Students, Teachers, and Schools: An Empirical Investigation of Key Design Parameters  \nDea Mulolli  \nHumRRO  \nE. C. Hedberg  \nWestat  \nMegan Bogia  \nFordham University  \nJessaca Spybrook  \nAmerican Institutes for Research, Western Michigan University  \nTiffany Berglund  \nRAND Corporation  \nFatih Unlu  \nAmazon  \nIsaac M. Opper  \nRAND Corporation  \nThe learning environment is a complex environment for conducting evaluations. Students, teachers, and schools all play a role in shaping the environment, which can lead to differences in student achievement. Hence designing an evaluation with the capacity to detect the effectiveness of an educational intervention on student achievement requires careful planning. A key part of the planning is a statistical power analysis that considers the multilevel nature of the design. This study provides empirical estimates of design parameters necessary for planning adequately powered cluster randomized trials that include the student, teacher, and school level and arefocused on reading, mathematics, or science achievement. The sample in our study includes administrative state datasets from Michigan, North Carolina, Kentucky, and Maryland for grades 3 through 8.  \nThe results showed that, with few exceptions, the variance in student test scores is larger between teachers within schools than between schools.  \nKeywords: Achievement, design parameters, experimental design, hierarchical linear modeling, intraclass correlation, program evaluation, teacher research  \nIntroduction  \nThe learning environment is a complex environment for conducting evaluations. It is critical to understand this environment to plan an evaluation that will successfully produce conclusive results. Typically, students are nested within one or more teachers, and teachers are nested within schools  \n(which in turn are nested within districts and states), implying a multilevel structure. This nested structure should be considered when modeling educational outcomes because teachers, schools, and districts contribute to variation in students’ outcomes. When planning evaluation studies of different educational interventions, it is important to think  \nCreative Commons Non Commercial CC BY-NC: This article is distributed under the terms of the Creative Commons  \nAttribution-NonCommercial 4.0 License ([https://creativecommons.org/licenses/by-nc/4.0/](https://creativecommons.org/licenses/by-nc/4.0/)) which permits non-commercial use,  \nreproduction and distribution of the work without further permission provided the original work is attributed as specified on the SAGE and Open Access pages ([https://us.sagepub.com/en-us/nam/open-access-at-sage](https://us.sagepub.com/en-us/nam/open-access-at-sage)).  \nMulolli et al.  \nabout the learning environment from a multilevel modeling perspective.  \nOver the past 15 years, researchers have started to assemble a literature base focused on empirically estimating the contribution of these various levels (e.g. students, and schools) to variation in student achievement. For example, we have learned that approximately 20 percent of the variance in student science achievement can be attributed to differences between schools and 80 percent of the variance can be attributed to differences between students within schools (Spybrook et al., 2016a) .  \nWhile this has been an active area of research, the teacher level has typically been absent from much of the literature, since many of the empirical analyses in the literature did not explicitly model the portion of outcome variance associated with teachers. The exceptions to this state of","cbCaiqTsSypQhkFV","https://ap.wps.com/l/cbCaiqTsSypQhkFV","pdf",678321,"English","# Introduction\n## Multilevel learning environment and nested structures\n## Teacher level inclusion in outcome variance\n## Empirical estimates and implications for ICC and power analysis\n## Data sources and study results\n# Keywords","[{\"question\":\"Why is statistical power analysis important for evaluations in this study?\",\"answer\":\"Because evaluations must account for the multilevel structure of students nested within teachers and teachers nested within schools to detect intervention effects with sufficient power.\"},{\"question\":\"What data sources and grade range were used in the analysis?\",\"answer\":\"The study uses administrative state datasets from Michigan, North Carolina, Kentucky, and Maryland for grades 3 through 8.\"},{\"question\":\"What does the study find about where student test score variance is largest?\",\"answer\":\"With few exceptions, student test score variance is larger between teachers within schools than between schools.\"}]","Improving the Design of Evaluations that Include Students, Teachers, and Schools - An Empirical Investigation of Key Design Parameters | PDF",1792069330]