[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-579085-105":59,"doc-detail-579085-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","comparing-zero-shot-large-language-model-prompting-with-human-coding-of-theory-concepts-in-student-essays","Comparing Zero-Shot Large Language Model Prompting with Human Coding of Theory Concepts in Student Essays","","The study compares zero-shot large language model (LLM) prompting with human qualitative coding of criminological theory concepts in college student essays. Four LLMs are evaluated against human-coded content-error judgments across multiple prompt types, theory coding dimensions, and criminological theories. Results show that LLM choice strongly determines human-AI correspondence, with Claude Sonnet 4 outperforming others and GPT 4.1 Mini performing worst. Prompt type has limited effect, while theory complexity and concept-definition accuracy drive error patterns.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/comparing-zero-shot-large-language-model-prompting-with-human-coding-of-theory-concepts-in-student-essays/579085/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/comparing-zero-shot-large-language-model-prompting-with-human-coding-of-theory-concepts-in-student-essays/579085.png","ImageObject",300,407,{"name":92,"@type":93},"Noah","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-15","2026-10-15",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",12,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What does the study compare between LLM prompting and human coding?","Question",{"text":113,"@type":114},"It compares qualitative coding accuracy for content errors in college student essays on criminological theories, using human-coded results as the reference.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"Which factor most strongly affects agreement between humans and LLMs?",{"text":118,"@type":114},"LLM choice significantly influences human-AI correspondence, with Claude Sonnet 4 showing the best overall performance and GPT 4.1 Mini the worst.",{"name":120,"@type":111,"acceptedAnswer":121},"How do prompt types and theory complexity affect results?",{"text":122,"@type":114},"Prompt type has minimal impact on performance, while LLMs perform better on more concise theories than on more complex ones.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},579085,1792172431,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":145,"read_time":146,"preview_page_count":4},8796095462418,"https://ap-avatar.wpscdn.com/avatar/80000253c1241d02b47?x-image-process=image/resize,m_fixed,w_180,h_180&k=1778826106357471780","Comparing Zero-Shot Large Language Model Prompting with Human Coding of Theory Concepts in Student Essays  \nShelley Keith University of Memphis Memphis , TN, USA [srkeith@memphis.edu](srkeith@memphis.edu)  \nPhilip I. Pavlik Jr. University of Memphis Memphis, TN, USA[ppavlik@memphis.edu](ppavlik@memphis.edu)  \nKristen L. Stives Seton Hall University South Orange, NJ, USA [kristen.stives@shu.edu](kristen.stives@shu.edu)  \nLaura Jean Kerr  \nGrand Valley State University Allendale, MI, USA [kerrlaur@gvsu.edu](kerrlaur@gvsu.edu)  \nRecent studies have explored the cost and time benefits of using artificial intelligence (AI), particularly large language models (LLMs), in coding student essays. While these models show promise, not enough is understood about the factors that affect how their qualitative coding performance compares to human coding. This study examines coding accuracy for content errors in college student essays on criminological theories by comparing human-coded results with outputs from four LLMs. We evaluated human-AI correlations, AI error, and AI bias across four LLMs, five prompt types, three theory content coding dimensions, and four criminological theories. Results indicate that LLM choice significantly influenced human-AI correspondence, with Claude Sonnet 4 exhibiting the best overall performance and GPT 4.1 Mini the worst. Prompt type had minimal impact on performance. Across models, error rates were lowest when identifying whether students listed a concept, and highest when assessing whether definitions were correct. LLMs performed better on concise theories than on more complex ones. The code is available at [https://github.com/imrryr/LLM-queries](https://github.com/imrryr/LLM-queries)  \n[Keywords:](Keywords: large language models)[ large language models](Keywords: large language models), automated essay scoring, qualitative coding, prompt engineering, student writing, criminological theory  \n1. INTRODUCTION  \nWhen assessing student writing, instructors must consider two important dimensions: conceptual ways to measure student comprehension and the point value (score) to assign to each facet of an assignment. The first task often involves determining what constitutes a correct or incorrect answer, which may be thought of as the process of coding, in which instructors assess the extent of students’ comprehension. The second task, scoring an assignment, involves assigning a quantitative value that represents the degree of student comprehension. Evaluating written assignments can be time-consuming and tedious, especially for instructors who aim to provide individualized feedback. Therefore, the aim of automated essay scoring (AES) is to reduce workloads for faculty, process high volumes of essays from high-stakes assessments, and provide consistency of evaluation for course credit (Kooli & Yusuf, 2025; Pack et al., 2024) .  \n286 Journal of Educational Data Mining, Volume 18, No 1, 2026  \nAES refers to the use of computer algorithms to evaluate and assign scores to written essays by analyzing features of the text, such as content relevance, organization, grammar, vocabulary, coherence, and sometimes argument quality, to approximate human rater judgments in a consistent and scalable way (Shermis & Wilson, 2021) . The present study builds on deductive qualitative content analysis by applying predefined theory-based concepts to student texts in a manner consistent with traditional qualitative methods (McClure et al., 2024) . Although traditional AES systems were not explicitly grounded in qualitative methods, both human coding and AES involve the systematic application of analytic criteria to text. While human coders apply these criteria based on professional judgment and expertise, AES systems make determinations algorithmically. In the context of theory-based evaluation of student essays, automated essay scoring can thus be understood as a specialized form of deductive coding. Contemporary AES frameworks simultan","cbCaicB5VR0aY9Lk","https://ap.wps.com/l/cbCaicB5VR0aY9Lk","pdf",803174,32,"English","# Introduction\n## Automated Essay Scoring and Coding\n## Role of Large Language Models in Deductive Coding\n## Prompt Engineering and Model Choice","[{\"question\":\"What does the study compare between LLM prompting and human coding?\",\"answer\":\"It compares qualitative coding accuracy for content errors in college student essays on criminological theories, using human-coded results as the reference.\"},{\"question\":\"Which factor most strongly affects agreement between humans and LLMs?\",\"answer\":\"LLM choice significantly influences human-AI correspondence, with Claude Sonnet 4 showing the best overall performance and GPT 4.1 Mini the worst.\"},{\"question\":\"How do prompt types and theory complexity affect results?\",\"answer\":\"Prompt type has minimal impact on performance, while LLMs perform better on more concise theories than on more complex ones.\"}]","Comparing Zero-Shot Large Language Model Prompting with Human Coding of Theory Concepts in Student Essays | PDF",1792034973,81]