[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-585381-105":59,"doc-detail-585381-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","a-human-in-the-loop-pipeline-for-multi-class-srl-classification","A Human-in-the-Loop Pipeline for Multi-Class SRL Classification","","Annotation of large educational datasets is time-intensive because constructs like self-regulated learning require theory-grounded expertise rather than surface cues. Recent attempts use large language models to scale labeling, but they overgeneralize from prompt examples and misclassify when contextual understanding is insufficient. This work studies human-in-the-loop machine learning to target the most labeling-relevant subset. Using multilingual DeBERTa classification and active learning, selective correction of weak labels improves performance and reduces LLM label misalignment for more responsible educational data annotation.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/a-human-in-the-loop-pipeline-for-multi-class-srl-classification/585381/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/a-human-in-the-loop-pipeline-for-multi-class-srl-classification/585381.png","ImageObject",300,407,{"name":92,"@type":93},"Aria Callaghan","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-31","2026-10-15",true,{"@type":102,"interactionType":103,"userInteractionCount":34},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"Why is annotating educational data for self-regulated learning difficult to scale?","Question",{"text":112,"@type":113},"Because SRL and related constructs are latent and must be inferred using theoretically grounded interpretation, requiring expert judgment beyond surface features. This makes annotation time-intensive and costly.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What problem do large language models pose for educational data annotation?",{"text":117,"@type":113},"LLMs can overgeneralize from prompt examples and misclassify due to limited context understanding. Their label reliability and theoretical fidelity are sensitive to prompting and task specification.",{"name":119,"@type":110,"acceptedAnswer":120},"How does the proposed human-in-the-loop pipeline improve multi-class classification performance?",{"text":121,"@type":113},"It combines active learning with selective correction of weak labels from LLMs, focusing annotation effort on the subset most beneficial for labeling or relabeling. Experiments with a multilingual DeBERTa-based classifier show improved performance versus a baseline trained on available human/LLM annotated data.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},585381,1792168099,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":34,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":39,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":143,"read_time":46,"preview_page_count":4},962084926284,"https://ap-avatar.wpscdn.com/davatar_29158cc5080c5b710cf443261637dec0","A Human-in-the-Loop Pipeline for Multi-Class SRL  \nClassification  \nAndreas Kissmehl  \nKTH Royal Institute of Technology Stockholm, Sweden  \n[kismehl@kth.se](kismehl@kth.se)  \nOlga Viberg  \nKTH Royal Institute of Technology Stockholm, Sweden  \n[oviberg@kth.se](oviberg@kth.se)  \nOlov Engwall  \nKTH Royal Institute of Technology Stockholm, Sweden  \n[engwall@kth.se](engwall@kth.se)  \nRichard Lee Davis  \nKTH Royal Institute of Technology Stockholm, Sweden  \n[rldavis@kth.se](rldavis@kth.se)  \nABSTRACT  \nAnnotation of big data is a nontrivial task for learning analytics scholars. Educational constructs drawing from learning psychology, such as self-regulated learning, require theorygrounded domain expertise and cannot be easily inferred from surface features, making annotation time-intensive and difficult to scale. Recent work aims to address this issue by using large language models (LLMs) to annotate data. While this approach can increase the amount of annotated data, evidence suggests that LLMs tend to overgeneralize, based on the examples included in the prompt cases, and misclassify due to a lack of context understanding. To address this, we investigate how human-in-the-loop machine learning strategies can be used to maximize the impact of human annotation by focusing efforts on a subset of data that would benefit most from labeling or relabeling. Utilized data include human-labeled, machine-labeled, and unlabeled student prompts from diverse educational contexts. We compare two active learning methods, loss weighting and their combination on the task of multi-class classification of student-generated text in AI-chatbot interactions. Experiments with a multilingual DeBERTa-based classifier indicate that combining human annotation with selective correction of weak labels can improve performance compared to a baseline trained on available annotated data annotated by humans and LLMs. The human-in-the-loop pipeline mitigates misalignment in labels produced by LLMs, providing a path forward to more responsible and rigorous use of LLMs in educational data annotation tasks.  \nKeywords  \nHuman-in-the-loop learning, active learning, annotation quality, large language models, self-regulated learning  \nAndreas Sebastian Kissmehl, Olga Viberg, Olov Engwall, and Richard Lee Davis. A Human-in-the-Loop Pipeline for Multi-Class SRL Classification. In Anthony Botelho, Maria Mercedes T. Rodrigo, Adish Singla, Hiroaki Ogata, Hyojeong So, and Young Hoan Cho (eds.) Proceedings of the 19th International Conference on Educational Data Mining, Seoul, Republic of Korea, June, 2026, pp. 400–407. International Educational Data Mining Society (2026) .  \n© 2026 Copyright is held by the author(s) . This work is distributed under the Creative Commons Attribution NonCommercial NoDerivatives 4.0 International (CC BY-NC-ND 4.0) license. [https://doi.org/10.5281/zenodo.21040133](https://doi.org/10.5281/zenodo.21040133)  \n1. INTRODUCTION  \nEducational constructs such as self-regulated learning (SRL), motivation, and help-seeking (e.g.,[20, 21, 11]) are widely used as proxies for studying students’ learning behaviors. However, these constructs are inherently latent and not directly observable in student interactions; instead, they must be inferred through a theoretically grounded interpretation. Researchers therefore need to operationalize them through observable indicators, such as discourse patterns, behavioral traces, or interaction features, connecting abstract theoretical constructs to empirical data. This process is necessarily interpretative and theory-laden, making annotation less a technical labeling exercise and more an expert-driven analytical task. Producing high-quality annotated datasets is thus essential but also highly time-intensive, costly, and difficult to scale, particularly when expertise in both learning theory and specific educational contexts is required [10] . The strong dependence on expert judgment and contextual understanding further lim","cbCaibAR5mO10Ylr","https://ap.wps.com/l/cbCaibAR5mO10Ylr","pdf",704631,"English","# Abstract\n# Keywords\n# 1. Introduction","[{\"question\":\"Why is annotating educational data for self-regulated learning difficult to scale?\",\"answer\":\"Because SRL and related constructs are latent and must be inferred using theoretically grounded interpretation, requiring expert judgment beyond surface features. This makes annotation time-intensive and costly.\"},{\"question\":\"What problem do large language models pose for educational data annotation?\",\"answer\":\"LLMs can overgeneralize from prompt examples and misclassify due to limited context understanding. Their label reliability and theoretical fidelity are sensitive to prompting and task specification.\"},{\"question\":\"How does the proposed human-in-the-loop pipeline improve multi-class classification performance?\",\"answer\":\"It combines active learning with selective correction of weak labels from LLMs, focusing annotation effort on the subset most beneficial for labeling or relabeling. Experiments with a multilingual DeBERTa-based classifier show improved performance versus a baseline trained on available human/LLM annotated data.\"}]","A Human-in-the-Loop Pipeline for Multi-Class SRL Classification | PDF",1792054564]