[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-579036-105":59,"doc-detail-579036-en":134},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":127,"head_meta":129,"extra_data":131,"updated_unix":133},105,"en","using-llms-to-identify-indicators-of-persistence-from-students-dialogues-with-a-pedagogical-agent","Using LLMs to Identify Indicators of Persistence from Students’ Dialogues with a Pedagogical Agent","","Conversational learning systems generate chat-log data that can predict academic success, yet identifying psychological constructs like persistence, self-efficacy, interest, perceived challenge, and prior knowledge has required costly manual coding. This study tests Large Language Models (LLMs) for automatically coding observable indicators from middle school mathematics conversation-based assessments (CBAs). It evaluates multiple ChatGPT4o configurations against human expert coders on data from 107 English learners, and analyzes reliability and construct alignment.",{"@graph":69,"@context":126},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/using-llms-to-identify-indicators-of-persistence-from-students-dialogues-with-a-pedagogical-agent/579036/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/using-llms-to-identify-indicators-of-persistence-from-students-dialogues-with-a-pedagogical-agent/579036.png","ImageObject",300,407,{"name":92,"@type":93},"Noah","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-15","2026-10-15",true,{"@type":102,"interactionType":103,"userInteractionCount":44},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118,122],{"name":109,"@type":110,"acceptedAnswer":111},"What psychological constructs does the study aim to code from student chat logs?","Question",{"text":112,"@type":113},"The study focuses on persistence, self-efficacy, interest, perceived challenge, and prior knowledge, using indicators observable in students’ conversational interactions within the CBA.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How do the researchers evaluate LLM performance for coding these constructs?",{"text":117,"@type":113},"They evaluate multiple ChatGPT4o configurations (including different temperature settings and model types) against human expert coders and assess reliability within and between LLM configurations and humans.",{"name":119,"@type":110,"acceptedAnswer":120},"What data was used, and who were the participants?",{"text":121,"@type":113},"The dataset includes over 10,000 student turns collected from 107 middle school students classified as English learners interacting with a conversation-based assessment (CBA).",{"name":123,"@type":110,"acceptedAnswer":124},"What key findings regarding model configuration and construct alignment are reported?",{"text":125,"@type":113},"Results show systematic patterns: constructs with moderate theoretical coherence benefited from higher temperatures, while well-defined constructs required deterministic settings; self-efficacy achieved the highest human–LLM alignment.","https://schema.org",{"og:url":83,"og:type":128,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":130,"canonical":83},"index,follow",{"doc_id":132,"site_id":62},579036,1792171258,{"code":4,"msg":5,"data":135},{"doc_id":132,"user_id":136,"nickname":92,"user_avatar":137,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":138,"file_id":139,"file_url":140,"file_type":141,"file_size":142,"view_count":44,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":143,"language":144,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":145,"faqs":146,"seo_title":147,"seo_description":67,"update_tm":148,"read_time":149,"preview_page_count":4},8796095462418,"https://ap-avatar.wpscdn.com/avatar/80000253c1241d02b47?x-image-process=image/resize,m_fixed,w_180,h_180&k=1778826106357471780","Using LLMs to Identify Indicators of Persistence from Students’ Dialogues with a Pedagogical Agent  \nTeresa M. Ober ETS  \nPrinceton, NJ, USA [tober@ets.org](tober@ets.org)  \nShan Zhang University of Florida Gainesville, FL, USA [zhangshan@ufl.edu](zhangshan@ufl.edu)  \nDiego Zapata-Rivera ETS  \nPrinceton, NJ, USA  \n[dzapata@ets.org](dzapata@ets.org)  \nNoah L. Schroeder University of Florida Gainesville, FL, USA  \n[schroedern@ufl.edu](schroedern@ufl.edu)  \nAnthony F. Botelho University of Florida Gainesville, FL, USA  \n[abotelho@coe.ufl.edu](abotelho@coe.ufl.edu)  \nConversational learning systems offer new opportunities to examine learning processes through chat log data. Constructs such as persistence, self-efficacy, interest, perceived challenge, and prior knowledge are known predictors of student performance but are challenging to detect at scale using traditional methods. This study explores the use of Large Language Models (LLMs) to automatically code indicators of these constructs from student chat logs collected through a conversation-based assessment (CBA) for middle school mathematics. Indicators included observable behaviors such as students’ expressions of challenge, help-seeking, goal-setting, and self-regulatory strategies evident in their conversational interactions within the CBA. We evaluated multiple configurations of ChatGPT4o, varying temperature settings (0, .3, .7, 1) and model types (mini vs. regular), against human expert coders. The dataset comprised over 10,000 student turns collected from 107 middle school students classified as English learners as they interact with the CBA. Reliability was assessed within and between LLM configurations and humans. Results reveal systematic patterns: constructs with moderate theoretical coherence benefited from higher temperatures, while well-defined constructs required deterministic settings. Self-efficacy showed the highest human-LLM alignment. These findings illustrate the challenges of measuring complex psychological constructs and highlight the promise of human-LLM collaboration to enhance qualitative coding efficiency and validity in educational research. Supplemental materials are available online here: [https://doi.org/10.17605/osf.io/s85ck](https://doi.org/10.17605/osf.io/s85ck).  \nKeywords: construct extraction, persistence, large language models (LLMs), model configuration, conversation-based assessment (CBA), human-LLM collaboration, qualitative analysis, educational data mining, construct validity, temperature settings  \n208 Journal of Educational Data Mining, Volume 18, No 1, 2026  \n1. INTRODUCTION  \nWhen students engage with conversational learning systems, their chat exchanges create a digital trail of learning processes that can predict academic success (Zapata-Rivera & Forsyth, 2022) . Critical psychological factors, including persistence, self-efficacy, interest, perceived challenge, and prior knowledge, consistently emerge as powerful predictors of learning outcomes (Nuutila et al., 2021; O ’Reilly et al., 2019) . Yet extracting these insights from student conversations has traditionally required painstaking manual analysis that cannot keep pace with today’s data-rich educational environments. While more traditional statistical and machine learning approaches have been used to analyze natural language data in educational settings (Botelho et al., 2023; Crossley et al., 2015; Dowell & Kovanović, 2022), large language models (LLMs) present a transformative opportunity to automatically detect these complex psychological constructs within student dialogue, potentially unlocking real-time understanding of learning processes at unprecedented scale. However, critical questions remain about how to harness LLMs for reliable, valid, and consistent analysis of educational constructs (Liu et al., 2025) .  \nCapturing authentic learning processes requires a delicate balance by gathering meaningful data without disrupting the natural flow of learning. Conversation","cbCaijqAcDB7wDFc","https://ap.wps.com/l/cbCaijqAcDB7wDFc","pdf",509898,36,"English","# 1. INTRODUCTION\n## 1.1. PERSISTENCE AS CENTRAL TO SELF-REGULATED LEARNING","[{\"question\":\"What psychological constructs does the study aim to code from student chat logs?\",\"answer\":\"The study focuses on persistence, self-efficacy, interest, perceived challenge, and prior knowledge, using indicators observable in students’ conversational interactions within the CBA.\"},{\"question\":\"How do the researchers evaluate LLM performance for coding these constructs?\",\"answer\":\"They evaluate multiple ChatGPT4o configurations (including different temperature settings and model types) against human expert coders and assess reliability within and between LLM configurations and humans.\"},{\"question\":\"What data was used, and who were the participants?\",\"answer\":\"The dataset includes over 10,000 student turns collected from 107 middle school students classified as English learners interacting with a conversation-based assessment (CBA).\"},{\"question\":\"What key findings regarding model configuration and construct alignment are reported?\",\"answer\":\"Results show systematic patterns: constructs with moderate theoretical coherence benefited from higher temperatures, while well-defined constructs required deterministic settings; self-efficacy achieved the highest human–LLM alignment.\"}]","Using LLMs to Identify Indicators of Persistence from Students’ Dialogues with a Pedagogical Agent | PDF",1792034883,91]