[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-413067-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-413067-en":130},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","leveraging-chatgpt-for-thematic-analysis-of-medical-best-practice-advisory-data","Leveraging ChatGPT for thematic analysis of medical best practice advisory data","","Objectives: Evaluate ChatGPT’s ability to perform thematic analysis of medical Best Practice Advisory (BPA) free-text comments and identify prompt engineering strategies that optimize performance. Materials and Methods: 778 BPA comments were analyzed from a Stanford Hospital pilot intervention, comparing prompt strategies on a 20% subsample, then using the best prompt for deductive coding and inductive analysis. Inter-rater reliability was assessed with Cohen’s Kappa. Results: Structured prompts plus calibration yielded substantial agreement with human coding (κ≈0.76–0.78), and produced themes closely aligned with human analysis.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/healthcare/","Healthcare",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/leveraging-chatgpt-for-thematic-analysis-of-medical-best-practice-advisory-data/413067/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/leveraging-chatgpt-for-thematic-analysis-of-medical-best-practice-advisory-data/413067.png","ImageObject",300,407,{"name":42,"@type":43},"jayni","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-03","2026-09-28",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",9,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What was the main objective of this study?","Question",{"text":63,"@type":64},"To evaluate ChatGPT’s ability to perform thematic analysis of medical BPA free-text comments and to identify prompt engineering strategies that improve performance.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How was the dataset and coding workflow organized?",{"text":68,"@type":64},"The study analyzed 778 BPA comments, tested prompt strategies on a 20% random subsample, then used the best-performing prompt for deductive coding of the full dataset followed by inductive analysis.",{"name":70,"@type":61,"acceptedAnswer":71},"How did the authors measure reliability between ChatGPT and human coders?",{"text":72,"@type":64},"Reliability was assessed as inter-rater reliability using Cohen’s Kappa (κ) comparing ChatGPT output with human coding for two categories.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},413067,1790986356,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,110,115,119,122,126],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":108,"slug":109},7,40,"healthcare",{"id":111,"doc_module":4,"doc_module_name":25,"category_name":112,"show_sort_weight":113,"slug":114},8,"Research & Report",30,"research-report",{"id":55,"doc_module":4,"doc_module_name":25,"category_name":116,"show_sort_weight":117,"slug":118},"Religion & Spirituality",20,"religion-spirituality",{"id":117,"doc_module":4,"doc_module_name":25,"category_name":120,"show_sort_weight":117,"slug":121},"World Cup","world-cup",{"id":123,"doc_module":4,"doc_module_name":25,"category_name":124,"show_sort_weight":123,"slug":125},10,"Lifestyle","lifestyle",{"id":127,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":97,"slug":129},19,"General","general",{"code":4,"msg":82,"data":131},{"doc_id":79,"user_id":132,"nickname":42,"user_avatar":133,"doc_module":4,"category_id":107,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":102,"language":139,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":12,"update_tm":143,"read_time":144,"preview_page_count":4},3985747859343,"https://ap-avatar.wpscdn.com/davatar_276721f389ce27ea32af1340a28f341c","JAMIA Open, 2025, 8(5), ooaf126 [https://doi.org/10.1093/jamiaopen/ooaf126](https://doi.org/10.1093/jamiaopen/ooaf126)[ ](https://doi.org/10.1093/jamiaopen/ooaf126)Research and Applications  \nResearch and Applications  \nLeveraging ChatGPT for thematic analysis of medical best practice advisory data  \nYejin Jeong, BA􀀃 , 1, Margaret Smith, MBA1, Robert J. Gallo , MD2,3, Lisa Marie Knowlton, MD, MPH4, Steven Lin, MD1,5, Lisa Shieh, MD, PhD5  \n1Stanford Healthcare AI Applied Research Team, Division of Primary Care and Population Health, Stanford University School of Medicine, Stanford, CA 94063, United States, 2Center for Innovation to Implementation, VA Palo Alto Health Care System, Menlo Park, CA 94025, United States, 3 Department of Health Policy, Stanford University, Stanford, CA 94305, United States, 4Section of Acute Care Surgery, Department of Surgery, Stanford University, Stanford, CA 94304, United States, 5 Department of Medicine, Stanford University School of Medicine, Stanford, CA 94305, United States  \n􀀃 Corresponding author: Yejin Jeong, BA, Stanford Healthcare AI Applied Research Team, Division of Primary Care and Population Health, Stanford University School of Medicine, 585 Broadway, Suite 800, Redwood City, CA 94063, United States ([yejeong@stanford.edu](yejeong@stanford.edu))  \nAbstract  \nObjectives: To evaluate ChatGPT’s ability to perform thematic analysis of medical Best Practice Advisory (BPA) free-text comments and identify prompt engineering strategies that optimize performance.  \nMaterials and Methods: We analyzed 778 BPA comments from a pilot AI-enabled clinical deterioration intervention at Stanford Hospital, categorized as reasons for deterioration (Category 1) and care team actions (Category 2) . Prompt engineering strategies (role, context specification, stepwise instructions, few-shot prompting, and dialogue-based calibration) were tested on a 20% random subsample to determine the bestperforming prompt. Using that prompt, ChatGPT conducted deductive coding on the full dataset followed by inductive analysis. Agreement with human coding was assessed as inter-rater reliability (IRR) using Cohen’s Kappa (κ) .  \nResults: With structured prompts and calibration, ChatGPT achieved substantial agreement with human coding (κ ¼ 0.76 for Category 1;κ ¼ 0.78 for Category 2) . Baseline agreement was higher for Category 1 than Category 2, reflecting differences in comment type and complexity, but calibration improved both. Inductive analysis yielded 9 themes, with ChatGPT-generated themes closely aligning with human coding. Discussion: ChatGPT can accelerate qualitative analysis, but its rigor depends heavily on prompt engineering. Key strategies included role and context specification, pulse-check calibration, and safeguard techniques, which enhanced reliability and reproducibility.  \nConclusion: This study demonstrates the feasibility of ChatGPT-assisted thematic analysis and introduces a structured approach for applying LLMs to qualitative analysis of clinical free-text data, underscoring prompt engineering as a methodological lever.  \nLay Summary  \nLarge language models like ChatGPT are increasingly being explored for research, but their usefulness depends on how they are guided. In this study, we assessed ChatGPT’s ability to analyze clinical free-text comments from a pilot program at Stanford Hospital. Our goal was not only to determine whether ChatGPT could perform the task but also to understand how different prompt designs affect reliability and performance. We compared ChatGPT’s results with those of a human researcher and experimented with prompt strategies such as adding role and context, providing examples, and including a calibration step before scaling up. Agreement with human coding was measured to identify the most effective approach. With the right combination of strategies, ChatGPT achieved substantial agreement with human coding and produced themes that closely matched human analysis. These find","cbCaiprA7lBQDPpc","https://ap.wps.com/l/cbCaiprA7lBQDPpc","pdf",657915,"English","# Abstract\n## Objectives\n## Materials and Methods\n## Results\n## Discussion and Conclusion\n# Lay Summary\n# Background and significance\n# Received, Revised, Accepted\n# Copyright and license","[{\"question\":\"What was the main objective of this study?\",\"answer\":\"To evaluate ChatGPT’s ability to perform thematic analysis of medical BPA free-text comments and to identify prompt engineering strategies that improve performance.\"},{\"question\":\"How was the dataset and coding workflow organized?\",\"answer\":\"The study analyzed 778 BPA comments, tested prompt strategies on a 20% random subsample, then used the best-performing prompt for deductive coding of the full dataset followed by inductive analysis.\"},{\"question\":\"How did the authors measure reliability between ChatGPT and human coders?\",\"answer\":\"Reliability was assessed as inter-rater reliability using Cohen’s Kappa (κ) comparing ChatGPT output with human coding for two categories.\"}]","Leveraging ChatGPT for thematic analysis of medical best practice advisory data | PDF",1790593666,15]