[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-1-en-105":3,"doc-seo-237997-105":53,"doc-detail-237997-en":126},{"code":4,"msg":5,"data":6},0,"success",[7,14,19,24,29,34,39,44,49],{"id":8,"doc_module":9,"doc_module_name":10,"category_name":11,"show_sort_weight":12,"slug":13},11,1,"Template","Presentations",90,"presentations",{"id":15,"doc_module":9,"doc_module_name":10,"category_name":16,"show_sort_weight":17,"slug":18},12,"Resumes",80,"resumes",{"id":20,"doc_module":9,"doc_module_name":10,"category_name":21,"show_sort_weight":22,"slug":23},14,"Invoices",70,"invoices",{"id":25,"doc_module":9,"doc_module_name":10,"category_name":26,"show_sort_weight":27,"slug":28},15,"Posters",60,"posters",{"id":30,"doc_module":9,"doc_module_name":10,"category_name":31,"show_sort_weight":32,"slug":33},16,"Social Media",50,"social-media",{"id":35,"doc_module":9,"doc_module_name":10,"category_name":36,"show_sort_weight":37,"slug":38},17,"Forms",40,"forms",{"id":40,"doc_module":9,"doc_module_name":10,"category_name":41,"show_sort_weight":42,"slug":43},18,"Letters",30,"letters",{"id":45,"doc_module":9,"doc_module_name":10,"category_name":46,"show_sort_weight":47,"slug":48},21,"Paper Templates",5,"papers-templates",{"id":50,"doc_module":9,"doc_module_name":10,"category_name":51,"show_sort_weight":4,"slug":52},158,"General","general-158",{"code":4,"msg":54,"data":55},"ok",{"site_id":56,"language":57,"slug":58,"title":59,"keywords":60,"description":61,"schema_data":62,"social_meta":119,"head_meta":121,"extra_data":123,"updated_unix":125},105,"en","asking-the-right-questions-benchmarking-large-language-models-in-the-development-of-clinical-consultation-templates","Asking the Right Questions - Benchmarking Large Language Models in the Development of Clinical Consultation Templates","","This study evaluates the capacity of large language models (LLMs) to generate structured clinical consultation templates for electronic consultation. Using 145 expert-crafted templates developed and routinely used by Stanford’s eConsult team, frontier models are assessed on producing clinically coherent, concise, and prioritized question schemas. A multi-agent pipeline with prompt optimization, semantic autograding, and prioritization analysis shows strong comprehensiveness for some models but persistent overlong outputs and misprioritization under length constraints, with larger degradation in narrative-driven specialties like psychiatry and pain medicine. Findings support improved structured exchange while motivating stronger evaluation for real-time physician communication, noting limitations from template specificity and concordance-based grading.",{"@graph":63,"@context":118},[64,80,101],{"@type":65,"itemListElement":66},"BreadcrumbList",[67,71,74,77],{"item":68,"name":69,"@type":70,"position":9},"https://docshare.wps.com","Home","ListItem",{"item":72,"name":10,"@type":70,"position":73},"https://docshare.wps.com/template/",2,{"item":75,"name":11,"@type":70,"position":76},"https://docshare.wps.com/template/presentations/",3,{"item":78,"name":59,"@type":70,"position":79},"https://docshare.wps.com/template/asking-the-right-questions-benchmarking-large-language-models-in-the-development-of-clinical-consultation-templates/237997/",4,{"url":78,"name":59,"@type":81,"image":82,"author":87,"headline":59,"publisher":90,"fileFormat":93,"inLanguage":57,"description":61,"dateModified":94,"datePublished":95,"encodingFormat":93,"isAccessibleForFree":96,"interactionStatistic":97},"DigitalDocument",{"url":83,"@type":84,"width":85,"height":86},"https://docshare.wps.com/thumbnails/asking-the-right-questions-benchmarking-large-language-models-in-the-development-of-clinical-consultation-templates/237997.png","ImageObject",442,249,{"name":88,"@type":89},"Taylor Morgan","Person",{"url":68,"name":91,"@type":92},"DocShare","Organization","application/pdf","2026-10-30","2026-09-11",true,{"@type":98,"interactionType":99,"userInteractionCount":8},"InteractionCounter",{"@type":100},"ViewAction",{"@type":102,"mainEntity":103},"FAQPage",[104,110,114],{"name":105,"@type":106,"acceptedAnswer":107},"What is the main goal of this study?","Question",{"text":108,"@type":109},"The study evaluates whether large language models can generate structured clinical consultation templates for electronic consultations that are coherent, concise, and properly prioritized.","Answer",{"name":111,"@type":106,"acceptedAnswer":112},"How were the language models evaluated?",{"text":113,"@type":109},"Models were tested using 145 expert-crafted Stanford eConsult templates, with a multi-agent pipeline that includes prompt optimization, semantic autograding, and prioritization analysis.",{"name":115,"@type":106,"acceptedAnswer":116},"What key limitations or failure modes were observed?",{"text":117,"@type":109},"Even when models achieve high comprehensiveness, they often produce excessively long templates and fail to prioritize the most clinically important questions within length constraints.","https://schema.org",{"og:url":78,"og:type":120,"og:title":59,"og:site_name":91,"og:description":61},"article",{"robots":122,"canonical":78},"index,follow",{"doc_id":124,"site_id":56},237997,1791746108,{"code":4,"msg":5,"data":127},{"doc_id":124,"user_id":128,"nickname":88,"user_avatar":129,"doc_module":9,"category_id":8,"category_name":11,"doc_title":59,"doc_description":61,"doc_content":130,"file_id":131,"file_url":132,"file_type":133,"file_size":134,"view_count":8,"is_deleted":4,"is_public":9,"is_downloadable":9,"audit_status":9,"page_count":35,"language":135,"language_code":57,"site_id":56,"html_lang":57,"table_of_contents":136,"faqs":137,"seo_title":138,"seo_description":61,"update_tm":139,"read_time":140},1099523885336,"https://ap-avatar.wpscdn.com/davatar_276721f389ce27ea32af1340a28f341c","Asking the Right Questions: Benchmarking Large Language Models in the Development of  \nClinical Consultation Templates  \nLiam G. McCoy†  \nDivision of Neurology, Faculty of Medicine and Dentistry, University of Alberta, Edmonton, AB, Canada; Department of Medicine, Beth Israel Deaconess Medical Center, Boston, MA, USA; Email: [lmccoy@ualberta.ca](lmccoy@ualberta.ca)  \nFateme Nateghi Haredasht† and Kanav Chopra  \nStanford Center for Biomedical Informatics Research Stanford, CA, USA  \n[Email: fnateghi@stanford.edu](Email: fnateghi@stanford.edu); [kanav@stanford.edu](kanav@stanford.edu)  \nDavid Wu  \nDepartment of Dermatology, Mass General Brigham, Harvard Medical School, Boston, MA, USA [Email: dwu@mgh.harvard.edu](Email: dwu@mgh.harvard.edu)  \nDavid JH Wu and Abass Conteh  \nDepartment of Radiation Oncology, Stanford Cancer Center, Palo Alto, CA, USA  \n[Email: davidjwu@stanford.edu](Email: davidjwu@stanford.edu); [aconteh@stanford.edu](aconteh@stanford.edu)  \nSarita Khemani and Saloni Kumar Maharaj  \nDivision of Hospital Medicine, Stanford University School of Medicine, Stanford, CA, USA Email: [skhemani@stanford.edu](skhemani@stanford.edu); [salonikumar@stanford.edu](salonikumar@stanford.edu)  \nArth Pahwa  \nDivision of Neurology, Faculty of Medicine and Dentistry, University of Alberta, Edmonton, AB, Canada [Email: apahwa@ualberta.ca](Email: apahwa@ualberta.ca)  \nVishnu Ravi  \nStanford Mussallem Center for Biodesign, Stanford University; Stanford University School of Medicine, Stanford, CA, USA Email: [vishnur@stanford.edu](vishnur@stanford.edu)  \nYingjie Weng  \nQuantitative Sciences Unit, Stanford University School of Medicine, Stanford, CA, USA [Email: yweng7@stanford.edu](Email: yweng7@stanford.edu)  \nLeah Rosengaus and Lena Giang  \nStanford Health Care, Stanford Medicine, Palo Alto, CA, USA  \n[Email: LRosengaus@stanfordhealthcare.org](Email: LRosengaus@stanfordhealthcare.org); [LGiang@stanfordhealthcare.org](LGiang@stanfordhealthcare.org)  \nKelvin Zhenghao Li  \nDepartment of Ophthalmology, Stanford Byers Eye Institute; Department of Ophthalmology, Tan Tock Seng Hospital; Centre of AI in Medicine, Lee Kong Chian School of Medicine, Nanyang Technological University Palo Alto, CA, USA; Singapore, Singapore [Email: kelvinli@stanford.edu](Email: kelvinli@stanford.edu)  \nOlivia Jee  \nDivision of Primary Care and Population Health, Stanford University School of Medicine, Palo Alto, CA, USA [Email: ojee@stanford.edu](Email: ojee@stanford.edu)  \nDaniel Shirvani  \nFaculty of Medicine, University of British Columbia, Vancouver, BC, Canada [Email: dshirv@student.ubc.ca](Email: dshirv@student.ubc.ca)  \nEthan Goh*  \nStanford Center for Biomedical Informatics Research Clinical Excellence Research Center, Stanford School of Medicine,  \nStanford, CA, USA Email: [ethangoh@stanford.edu](ethangoh@stanford.edu)  \nJonathan H. Chen*  \nStanford Center for Biomedical Informatics Research Division of Hospital Medicine, Stanford School of Medicine; Clinical Excellence Research Center, Stanford School of Medicine; Department of Medicine, Stanford University, Stanford, CA, USA [Email: jonc101@stanford.edu](Email: jonc101@stanford.edu)  \nAbstract:  \nThis study evaluates the capacity of large language models (LLMs) to generate structured clinical consultation templates for electronic consultation. Using 145 expert-crafted templates developed and routinely used by Stanford’s eConsult team, we assess frontier models—including o3, GPT-4o, Kimi K2, Claude 4 Sonnet, Llama 3 70B, and Gemini 2.5 Pro—for their ability to produce  \nclinically coherent, concise, and prioritized clinical question schemas. Through a multi-agent pipeline combining prompt optimization, semantic autograding, and prioritization analysis, we show that while models like o3 achieve high comprehensiveness (up to 92.2%), they consistently generate excessively long templates and fail to correctly prioritize the most clinically important questions under length constraints. Performance varies acr","cbCaiveq7OyGB7Pa","https://ap.wps.com/l/cbCaiveq7OyGB7Pa","pdf",964557,"English","# 1. Introduction\n## Consultation structure and communication risks\n## Synchronous vs. asynchronous e-consult workflows\n## Reliance on initial clinician-provided information","[{\"question\":\"What is the main goal of this study?\",\"answer\":\"The study evaluates whether large language models can generate structured clinical consultation templates for electronic consultations that are coherent, concise, and properly prioritized.\"},{\"question\":\"How were the language models evaluated?\",\"answer\":\"Models were tested using 145 expert-crafted Stanford eConsult templates, with a multi-agent pipeline that includes prompt optimization, semantic autograding, and prioritization analysis.\"},{\"question\":\"What key limitations or failure modes were observed?\",\"answer\":\"Even when models achieve high comprehensiveness, they often produce excessively long templates and fail to prioritize the most clinically important questions within length constraints.\"}]","Asking the Right Questions - Benchmarking Large Language Models in the Development of Clinical Consultation Templates | PDF",1789129249,6]