[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-579098-105":3,"doc-detail-579098-en":81,"detail-sidebar-cat-0-en-105":99},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","human-ai-collaboration-for-qualitative-analysis-in-participatory-design-refining-the-writing-analytics-tool","Human-AI Collaboration for Qualitative Analysis in Participatory Design - Refining the Writing Analytics Tool","","This study introduces a hybrid human-AI workflow for qualitative data analysis within the participatory design of the Writing Analytics Toolkit (WAT), an open-source system that delivers formative writing feedback using natural language processing. WAT includes a classroom implementation (WAT-C) for instructional support and a researcher implementation (WAT-R) for analytics and validation. Using focus groups with nine experienced instructors, the authors apply a secure GPT-4o retrieval-augmented pipeline to generate inductive codes and themes that researchers iteratively review and refine.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/human-ai-collaboration-for-qualitative-analysis-in-participatory-design-refining-the-writing-analytics-tool/579098/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/human-ai-collaboration-for-qualitative-analysis-in-participatory-design-refining-the-writing-analytics-tool/579098.png","ImageObject",300,407,{"name":42,"@type":43},"Noah","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-15","2026-10-15",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",14,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What is the Writing Analytics Toolkit (WAT) and how is it organized?","Question",{"text":63,"@type":64},"WAT is an open-source platform providing formative writing analytics using natural language processing. It has two implementations: WAT Classroom (WAT-C) for instructional use and WAT Researcher (WAT-R) for analytic and validation workflows.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How does the study use AI to support qualitative analysis?",{"text":68,"@type":64},"The study integrates GPT-4o into a secure retrieval-augmented system to generate inductive codes and preliminary themes from transcripts. Researchers then iteratively review, critique, refine, and synthesize these outputs.",{"name":70,"@type":61,"acceptedAnswer":71},"What design priorities were identified and how did they affect revisions to WAT-C?",{"text":72,"@type":64},"The study identifies four priorities: clearer and more concise feedback, increased instructor customization, reduced administrative burden, and a simplified user interface. These findings guided revisions such as a redesigned feedback interface, customizable metric targets, LMS integration, and a more intuitive layout.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},579098,1792190888,{"code":4,"msg":82,"data":83},"success",{"doc_id":79,"user_id":84,"nickname":42,"user_avatar":85,"doc_module":4,"category_id":86,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":87,"file_id":88,"file_url":89,"file_type":90,"file_size":91,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":92,"language":93,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":94,"faqs":95,"seo_title":96,"seo_description":12,"update_tm":97,"read_time":98,"preview_page_count":4},8796095462418,"https://ap-avatar.wpscdn.com/avatar/80000253c1241d02b47?x-image-process=image/resize,m_fixed,w_180,h_180&k=1778826106357471780",8,"Human-AI Collaboration for Qualitative Analysis in Participatory Design: Refining the Writing Analytics Tool  \nAndrew Potter Arizona State University Tempe, United States  \n[ahpotter@asu.edu](ahpotter@asu.edu)  \nNishad A. Patne Arizona State University Tempe, United States  \n[npatne@asu.edu](npatne@asu.edu)  \nIshrat Ahmed Arizona State University Tempe, United States  \n[Ishrat.Ahmed.1@asu.edu](Ishrat.Ahmed.1@asu.edu)  \nRezwana Islam Arizona State University Tempe, United States  \n[rislam11@asu.edu](rislam11@asu.edu)  \nLaura A. Allen University of Minnesota Minneapolis, United States [lallen@umn.edu](lallen@umn.edu)  \nZeinab Serhan Arizona State University Tempe, United States  \n[zserhan@asu.edu](zserhan@asu.edu)  \nPüren Öncel  \nUniversity of Minnesota Minneapolis, United States  \n[oncel001@umn.edu](oncel001@umn.edu)  \nTracy Arner  \nArizona State University Tempe, United States  \n[tarner@asu.edu](tarner@asu.edu)  \nRod D. Roscoe Arizona State University Tempe, United States  \n[rod.roscoe@asu.edu](rod.roscoe@asu.edu)  \nScott A. Crossley Vanderbilt University Nashville, United States  \n[scott.crossley@vanderbilt.edu](scott.crossley@vanderbilt.edu)  \nDanielle S. McNamara Arizona State University Tempe, United States  \n[dsmcnamara@asu.edu](dsmcnamara@asu.edu)  \nThis study introduces a hybrid human-AI workflow to qualitative data analysis within the participatory design of the Writing Analytics Toolkit (WAT), an open-source platform that provides formative feedback on student writing using natural language processing. The toolkit includes a classroom-facing implementation (WAT Classroom; WAT-C), designed to support instruction, and a researcher-facing implementation (WAT Researcher; WAT-R), designed to support analytic and validation workflows. Nine experienced college writing instructors (with 97 cumulative years of teaching) participated in focus group sessions to evaluate an early  \n113 Journal of Educational Data Mining, Volume 18, No 1, 2026  \nprototype of the classroom version of WAT (WAT-C), offering formative input on usability, instructional alignment, and feedback clarity. To analyze the resulting qualitative data, we employed a novel AI-augmented analytic process: GPT-4o, integrated within a secure, retrieval-augmented system, to generate inductive codes and preliminary themes from transcripts. These AI-generated outputs were iteratively reviewed, critiqued, refined, and synthesized by researchers, supporting both analytical scalability and interpretive rigor. This human-AI partnership enabled efficient thematic exploration while preserving methodological transparency and researcher judgment. Findings from both qualitative and complementary survey data identified four key design priorities: (1) clearer, more concise feedback,(2) increased instructor customization,(3) reduced administrative burden, and (4) a simplified user interface. These insights directly informed subsequent revisions to WAT-C, including a redesigned feedback interface, customizable metric targets, learning management system integration, and a more intuitive layout. This work illustrates how large language models (LLMs) can support inductive qualitative analysis within participatory design workflows. Moreover, results demonstrate how this workflow can inform iterative educational technology development. Implications include the need to ensure ethical oversight, researcher-led interpretation, and alignment with instructional priorities when incorporating AI into the design of educational technologies.  \nKeywords: writing analytics, participatory design, generative AI, qualitative data analysis, natural language processing, educational technology  \n1. INTRODUCTION  \nAutomated writing evaluation (AWE) and writing analytics tools offer scalable support for both teaching and learning of writing by providing students with immediate, formative feedback to guide revision (Potter et al., 2025; Strobl et al., 2019; Wilson & MacArthur, 2024) . However, developing e","cbCailrctzbii9wt","https://ap.wps.com/l/cbCailrctzbii9wt","pdf",790063,43,"English","# Introduction\n## Automated writing evaluation and participatory design needs\n## Study focus within Writing Analytics Toolkit (WAT)\n## Contributions and workflow overview","[{\"question\":\"What is the Writing Analytics Toolkit (WAT) and how is it organized?\",\"answer\":\"WAT is an open-source platform providing formative writing analytics using natural language processing. It has two implementations: WAT Classroom (WAT-C) for instructional use and WAT Researcher (WAT-R) for analytic and validation workflows.\"},{\"question\":\"How does the study use AI to support qualitative analysis?\",\"answer\":\"The study integrates GPT-4o into a secure retrieval-augmented system to generate inductive codes and preliminary themes from transcripts. Researchers then iteratively review, critique, refine, and synthesize these outputs.\"},{\"question\":\"What design priorities were identified and how did they affect revisions to WAT-C?\",\"answer\":\"The study identifies four priorities: clearer and more concise feedback, increased instructor customization, reduced administrative burden, and a simplified user interface. These findings guided revisions such as a redesigned feedback interface, customizable metric targets, LMS integration, and a more intuitive layout.\"}]","Human-AI Collaboration for Qualitative Analysis in Participatory Design - Refining the Writing Analytics Tool | PDF",1792034987,108,{"code":4,"msg":82,"data":100},[101,105,109,113,118,123,128,131,136,139,143],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":102,"show_sort_weight":103,"slug":104},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":106,"show_sort_weight":107,"slug":108},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":110,"show_sort_weight":111,"slug":112},"Exam",70,"exam",{"id":114,"doc_module":4,"doc_module_name":25,"category_name":115,"show_sort_weight":116,"slug":117},5,"Comic",60,"comic",{"id":119,"doc_module":4,"doc_module_name":25,"category_name":120,"show_sort_weight":121,"slug":122},6,"Technology",50,"technology",{"id":124,"doc_module":4,"doc_module_name":25,"category_name":125,"show_sort_weight":126,"slug":127},7,"Healthcare",40,"healthcare",{"id":86,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":129,"slug":130},30,"research-report",{"id":132,"doc_module":4,"doc_module_name":25,"category_name":133,"show_sort_weight":134,"slug":135},9,"Religion & Spirituality",20,"religion-spirituality",{"id":134,"doc_module":4,"doc_module_name":25,"category_name":137,"show_sort_weight":134,"slug":138},"World Cup","world-cup",{"id":140,"doc_module":4,"doc_module_name":25,"category_name":141,"show_sort_weight":140,"slug":142},10,"Lifestyle","lifestyle",{"id":144,"doc_module":4,"doc_module_name":25,"category_name":145,"show_sort_weight":114,"slug":146},19,"General","general"]