[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-459051-105":59,"doc-detail-459051-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","automated-safety-plan-scoring-in-outpatient-mental-health-settings-using-large-language-models-exploratory-study","Automated Safety Plan Scoring in Outpatient Mental Health Settings Using Large Language Models: Exploratory Study","","The safety planning intervention (SPI) helps patients reduce suicide risk through a written plan whose quality strongly influences effectiveness. High-quality, personalized plans are hard to measure because assessing fidelity is labor-intensive and clinicians rarely receive specific, actionable feedback. This study develops the Safety Plan Fidelity Rater, an automated tool using three large language models to evaluate plan quality. The work tests model performance on deidentified outpatient plans and compares step-level scoring approaches.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":35,"@type":76,"position":81},"https://docshare.wps.com/document/healthcare/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/automated-safety-plan-scoring-in-outpatient-mental-health-settings-using-large-language-models-exploratory-study/459051/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/automated-safety-plan-scoring-in-outpatient-mental-health-settings-using-large-language-models-exploratory-study/459051.png","ImageObject",300,407,{"name":92,"@type":93},"mieayamfan","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-30","2026-09-30",true,{"@type":102,"interactionType":103,"userInteractionCount":24},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What is the Safety Planning Intervention (SPI) and what makes plan quality important?","Question",{"text":112,"@type":113},"SPI is a suicide prevention intervention that guides patients to create a written safety plan. More complete, personalized, and specific plans are more effective at reducing suicide risk.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does this study evaluate written safety plan quality?",{"text":117,"@type":113},"The study develops the Safety Plan Fidelity Rater, which uses three large language models (GPT-4, LLaMA 3, and o3-mini) to score key plan steps such as warning signs, coping strategies, environmental safety, and reasons for living.",{"name":119,"@type":110,"acceptedAnswer":120},"Which large language models performed best for safety plan scoring?",{"text":121,"@type":113},"LLaMA 3 and o3-mini outperformed GPT-4, with recommendations for different step-specific scoring systems based on weighted F1-scores.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},459051,1791053171,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":34,"category_name":35,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":24,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":52,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":143,"read_time":144,"preview_page_count":4},962090893677,"https://ap-avatar.wpscdn.com/davatar_3d24733baf745e90a7e4bdd5f77d97b2","JMIR MENTAL HEALTH Donnelly et al  \nOriginal Paper  \nAutomated Safety Plan Scoring in Outpatient Mental Health Settings Using Large Language Models: Exploratory Study  \n\n| Hayoung K Donnelly 1,2 , PhD; Gregory K Brown1 , PhD; Kelly L Green1 , PhD; Ugurcan Vurgun2 , PhD; Sy Hwang2 , MS; Emily Schriver2,3 , MS; Michael Steinberg 1 , MA; Megan E Reilly 1 , MPH; Haitisha Mehta4 , MA; Christa Labouliere4 , PhD; Maria A Oquendo 1 , MD, PhD; David Mandell 1 , SCD; Danielle L Mowery2,5 , PhD |\n| --- |\n| 1Department of Psychiatry, University of Pennsylvania, Philadelphia, PA, United States 2Institute for Biomedical Informatics, University of Pennsylvania, Philadelphia, PA, United States 3University of Pennsylvania Health System, Philadelphia, PA, United States\u003Cbr>4Department of Psychiatry, Columbia University, New York, NY, United States\u003Cbr>5Department of Biostatistics, Epidemiology and Informatics, University of Pennsylvania, Philadelphia, PA, United States\u003Cbr>Corresponding Author:\u003Cbr>Danielle L Mowery, PhD\u003Cbr>Institute for Biomedical Informatics\u003Cbr>University of Pennsylvania 3700 Hamilton Walk Philadelphia, PA 19104 United States\u003Cbr>Phone: 1-215-746-6677\u003Cbr>Email: [dlmowery@pennmedicine.upenn.edu](dlmowery@pennmedicine.upenn.edu)\u003Cbr>Abstract |\n| Background: The safety planning intervention (SPI) is a suicide prevention intervention that results in a written plan to help patients reduce suicide risk. High-quality safety plans—that is, those that are the most complete, personalized, and specific—are more effective in reducing suicide risk. Measuring SPI quality is labor-intensive, which means that clinicians rarely get specific, actionable feedback on their use of the SPI.\u003Cbr>Objective: This study aimed to develop the Safety Plan Fidelity Rater, an automated tool that assesses the quality of written safety plans leveraging 3 large language models (LLMs)—GPT-4, LLaMA 3, and o3-mini.\u003Cbr>Methods: Using 266 deidentified safety plans from outpatient mental health settings in New York, LLMs analyzed four key steps: warning signs, internal coping strategies, making the environment safe, and reasons for living. We compared the predictive performance of the three LLMs, optimizing scoring systems, prompts, and parameters.\u003Cbr>Results: Findings showed that LLaMA 3 and o3-mini outperformed GPT-4, with different step-specific scoring systems recommended based on weighted F1-scores .\u003Cbr>Conclusions: These findings highlight LLMs’ potential to provide clinicians with timely and accurate feedback on safety plan quality, which could greatly improve its implementation in community practice.\u003Cbr>JMIR Ment Health 2026;13:e79010; doi:  10.2196/79010 |\n\nKeywords: suicide; mental health informatics; generative AI; clinician support; patient-reported data; artificial intelligence  \nIntroduction  \nThe safety planning intervention (SPI) is a widely used, evidence-based intervention to prevent suicide [1] . It is designed to help patients develop awareness of their personal warning signs and pre-plan specific strategies to use to prevent and manage acute suicidal crises [1] . The written safety plan that results from SPI has six steps: (1) identifying internal warning signs of an impending acute suicidal crisis;  \n(2) identifying distracting activities the individual can do by themselves; (3) identifying people and social settings that can be used for distraction; (4) identifying friends and family that the individual can reach out to for help; (5) identifying professionals and professional resources the individual can reach out to for help; and (6) developing an action plan to reduce access to lethal means . In some cases, the written safety plan includes an optional Step 7, which identifies the patient’s reasons for living. SPI reduces suicidal behavior  \n[https://mental.jmir.org/2026/1/e79010](https://mental.jmir.org/2026/1/e79010) JMIR Ment Health 2026 | vol. 13 | e79010 | p. 1  \n(page number not for citation purposes)  \nJMIR MENTAL HEALTH Donnelly et al  \nfol","cbCaic9tE1KGWzOa","https://ap.wps.com/l/cbCaic9tE1KGWzOa","pdf",1005901,"English","# Abstract\n# Background\n# Objective\n# Methods\n# Results\n# Conclusions\n# Keywords\n# Introduction","[{\"question\":\"What is the Safety Planning Intervention (SPI) and what makes plan quality important?\",\"answer\":\"SPI is a suicide prevention intervention that guides patients to create a written safety plan. More complete, personalized, and specific plans are more effective at reducing suicide risk.\"},{\"question\":\"How does this study evaluate written safety plan quality?\",\"answer\":\"The study develops the Safety Plan Fidelity Rater, which uses three large language models (GPT-4, LLaMA 3, and o3-mini) to score key plan steps such as warning signs, coping strategies, environmental safety, and reasons for living.\"},{\"question\":\"Which large language models performed best for safety plan scoring?\",\"answer\":\"LLaMA 3 and o3-mini outperformed GPT-4, with recommendations for different step-specific scoring systems based on weighted F1-scores.\"}]","Automated Safety Plan Scoring in Outpatient Mental Health Settings Using Large Language Models: Exploratory Study | PDF",1790755063,25]