[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-577868-105":59,"doc-detail-577868-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","using-large-language-models-to-evaluate-ethical-persuasion-text-a-measurement-modeling-approach","Using large language models to evaluate ethical persuasion text - A measurement modeling approach","","As AI becomes pervasive in educational assessment workflows, defensible measurement procedures are needed to ensure ethical, valid, and fair evaluation. This study integrates measurement principles into an ethical reasoning performance assessment where large language models (LLMs) act as raters, producing measures for LLM-generated ethical persuasion responses, designed prompts, and individual learners. Measurement quality indicators are used as guardrails to reduce AI risks such as hallucinations and errors, enabling effective and personalized learning.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/using-large-language-models-to-evaluate-ethical-persuasion-text-a-measurement-modeling-approach/577868/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/using-large-language-models-to-evaluate-ethical-persuasion-text-a-measurement-modeling-approach/577868.png","ImageObject",300,407,{"name":92,"@type":93},"Emma Wilson","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-16","2026-10-15",true,{"@type":102,"interactionType":103,"userInteractionCount":34},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What is the main purpose of the study?","Question",{"text":112,"@type":113},"The study demonstrates how measurement theory can evaluate ratings generated by large language models in a performance assessment focused on ethical persuasion.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How are large language models used in the assessment?",{"text":117,"@type":113},"LLMs serve as raters who score sample responses that have specifically designed or selected characteristics, and the rating quality is evaluated from a psychometric perspective.",{"name":119,"@type":110,"acceptedAnswer":120},"What kinds of research questions does the study address?",{"text":121,"@type":113},"It examines how measurement models evaluate psychometric quality, whether ratings show useful model-data fit and scale functioning, how settings and prompt specifications affect ratings, and whether bias appears in non-invariant calibrations across subgroups.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},577868,1792622549,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":34,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":26,"preview_page_count":4},3848291630094,"https://eur-avatar.wpscdn.com/davatar_085a072bc5b1113ac321206ff7593b45","Using large language models to evaluate ethical persuasion text: A measurement modeling approach  \nMatt Barney1, Stefanie A. Wind2*, Vaishak Krishna3  \n[1](1TruMind.ai)[TruMind.ai](1TruMind.ai), USA  \n2University of Alabama, Department of Educational Studies, Tuscaloosa, AL, USA 3Apple, Inc., USA  \nARTICLE HISTORY  \nReceived: Oct. 7, 2025  \nAccepted: Dec. 7, 2025  \nKEYWORDS  \nPerformance assessment, Large language models, Rater-mediated assessment, Rasch measurement theory.  \nAbstract: As AI becomes prevalent in all stages of assessment procedures, it is essential to develop procedures to ensure that its use supports ethical and psychometrically defensible measurement. In this study, we consider how measurement principles can be directly incorporated into an ethical reasoning performance assessment in which Large Language Models (LLMs) serve as raters. We demonstrate how a measurement approach can be used to obtain defensible measures of LLM-generated text related to ethics, prompts designed to elicit text-based ethical persuasion responses, and individual learners. We demonstrate how measurement quality indicators can serve as guardrails to help mitigate potential AI-related risks that can impact learners, such as hallucinations or errors. We describe a novel approach to designing, implementing, and evaluating performance assessments with AI, with the goal of enabling effective personalized learning experiences  \nINTRODUCTION  \nThe emergence of Large Language Models (LLMs) presents exciting opportunities and important challenges related to educational measurement. On one hand, these artificial intelligence (AI) systems have the potential to revolutionize the way we develop, administer, and interpret assessments, enabling more efficient, accurate, and personalized evaluation of student learning (Chia et al., 2023) . The use of LLMs in educational measurement also raises concerns related to explainability, as well as the presence of hallucinations and bias in AI-generated content (Schwartz et al., 2022; Solaiman et al., 2023) . These issues pose significant challenges and opportunities for future research to ensure validity, reliability, and fairness of AI-assisted assessments.  \nAn active area of research is the use of LLMs as automated raters to evaluate examinee performances (e.g., Mizumoto & Eguchi, 2023) . Although automated scoring is not new (Page, 1966), the recent advancements and availability of LLMs have inspired increased interest and  \n*Corresponding author at: Stefanie A. WIND 􀀍 [swind@ua.edu](swind@ua.edu) 􀁽 Box 870321 Tuscaloosa, AL 35487, USA  \nThe copyright of the published article belongs to its author under CC BY 4.0 license. To view a copy of this licence, visit [http://creativecommons.org/licenses/by/4.0/](http://creativecommons.org/licenses/by/4.0/)  \n[e-ISSN: 2148-7456](e-ISSN: 2148-7456)  \napplications of automated scoring in both research and practice, particularly within the context of writing assessment (e.g., Pfau et al., 2023; Ramesh & Sanampudi, 2021; Seßler et al., 2025) . As in traditional rater-mediated assessments with human raters (Engelhard & Wind, 2018), LLM output is affected by a range of variables (e.g., prompt quality) that must be understood and managed to ensure the validity, reliability, and fairness of the results. Researchers have evaluated the quality of LLM ratings, often focusing on comparability with human raters (e.g., Floden, 2025; Kim et al., 2024; Wetzler et al., 2024; Yavus et al., 2024; Yun, 2023) as a source of validity evidence. Studies are less common that have used measurement modeling techniques to evaluate LLM ratings (e.g., Chan et al., 2023; Yamashita, 2024). As we discuss later in the manuscript, analyses informed by a measurement framework can provide researchers with clear guidelines for evaluating AI-assisted assessment processes beyond indicators of reliability and agreement. To the best ofour knowledge, this application of measurement techniques to evaluate ","cbCain28HL2TM5Pm","https://ap.wps.com/l/cbCain28HL2TM5Pm","pdf",745530,24,"English","# Introduction\n## Purpose and research questions\n# Measurement approach for LLM raters\n## Psychometric quality and model fit\n## Prompt specifications and rating bias","[{\"question\":\"What is the main purpose of the study?\",\"answer\":\"The study demonstrates how measurement theory can evaluate ratings generated by large language models in a performance assessment focused on ethical persuasion.\"},{\"question\":\"How are large language models used in the assessment?\",\"answer\":\"LLMs serve as raters who score sample responses that have specifically designed or selected characteristics, and the rating quality is evaluated from a psychometric perspective.\"},{\"question\":\"What kinds of research questions does the study address?\",\"answer\":\"It examines how measurement models evaluate psychometric quality, whether ratings show useful model-data fit and scale functioning, how settings and prompt specifications affect ratings, and whether bias appears in non-invariant calibrations across subgroups.\"}]","Using large language models to evaluate ethical persuasion text - A measurement modeling approach | PDF",1792032567]