[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-506013-105":59,"doc-detail-506013-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","a-framework-for-human-evaluation-of-large-language-models-in-healthcare-derived-from-literature-review","A Framework for Human Evaluation of Large Language Models in Healthcare - Derived from Literature Review","","Generative AI, especially large language models (LLMs), is increasingly used in healthcare, making human evaluation essential to complement traditional automated metrics. Existing human evaluation work remains hindered by methods that are time-consuming, cumbersome, and inconsistently standardized. This literature review synthesizes studies from January 2018 to February 2024, covering evaluation dimensions, samples, evaluator recruitment, frameworks, metrics, procedures, and statistical analyses across medical specialties. It proposes QUEST to support reliable, generalizable, and practical human assessment of LLM outputs in healthcare.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":35,"@type":76,"position":81},"https://docshare.wps.com/document/healthcare/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/a-framework-for-human-evaluation-of-large-language-models-in-healthcare-derived-from-literature-review/506013/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/a-framework-for-human-evaluation-of-large-language-models-in-healthcare-derived-from-literature-review/506013.png","ImageObject",300,407,{"name":92,"@type":93},"8796093062539","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-22","2026-10-04",true,{"@type":102,"interactionType":103,"userInteractionCount":24},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"Why is human evaluation important for large language models in healthcare?","Question",{"text":112,"@type":113},"Because automated evaluations alone may not ensure the clinical utility, safety, reliability, and effectiveness required for healthcare deployment. Human evaluation addresses gaps that metrics cannot cover.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What scope and period does the literature review cover?",{"text":117,"@type":113},"The review follows PRISMA guidelines and includes publications from January 2018 to February 2024.",{"name":119,"@type":110,"acceptedAnswer":120},"What is the QUEST framework used for?",{"text":121,"@type":113},"QUEST provides clear evaluation dimensions and detailed guidelines for human assessment of LLMs in healthcare, aiming to improve reliability, generalizability, and applicability across applications.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},506013,1791428878,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":66,"doc_module":4,"category_id":34,"category_name":35,"doc_title":65,"doc_description":67,"doc_content":133,"file_id":134,"file_url":135,"file_type":136,"file_size":137,"view_count":24,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":138,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":143,"read_time":144,"preview_page_count":4},8796093062539,"A Framework for Human Evaluation of Large Language Models in Healthcare  \nDerived from Literature Review  \n* *  \nThomas Yu Chow Tam, MS1 , Sonish Sivarajkumar, BS2 , Sumit Kapoor, MD3, Alisa V Stolyar1, Katelyn Polanska, BS1, Karleigh R McCarthy, BS1, Hunter Osterhoudt, MS1, Xizhi Wu, MS1, Shyam Visweswaran, MD, PhD4, Sunyang Fu, PhD5, Piyush Mathur, MD6, Giovanni E. Cacciamani, MD7, Cong Sun, PhD8, Yifan Peng, PhD8, Yanshan  \nWang,PhD1,2,4,9,10  \n†  \n1Department ofHealth Information Management, University ofPittsburgh, Pittsburgh, PA, USA; 2Intelligent Systems Program, University ofPittsburgh, Pittsburgh, PA, USA; 3Department of Critical Care Medicine, University ofPittsburgh Medical Center, Pittsburgh, PA, USA;4Department ofBiomedical Informatics, University ofPittsburgh, Pittsburgh, PA, USA; 5Department ofClinical and Health Informatics, Center for Translational AI Excellence and Applications in Medicine, University ofTexas Health Houston, Houston, TX, USA; 6Department of Anesthesiology, Cleveland Clinic, Cleveland, OH, USA; 7Department of Urology, Keck School ofMedicine, University ofSouthern California, Los Angeles, CA, USA; 8Department of Population Health Sciences, Weill Cornell Medicine, New York, NY, USA; 9 Clinical and Translational Science Institute, University ofPittsburgh, Pittsburgh, PA; 10Hillman Cancer Center, University ofPittsburgh Medical Center, Pittsburgh, PA;  \n* co-first authors  \n† corresponding author: Yanshan Wang, [yanshan.wang@pitt.edu](yanshan.wang@pitt.edu)  \nAbstract  \nWith generative artificial intelligence (AI) , particularly large language models ( LLMs) , continuing to make inroads in healthcare, it is critical to supplement traditional automated evaluations with human evaluations. Understanding and evaluating the output of LLMs is essential to assuring safety, reliability, and effectiveness. However, human evaluation's cumbersome,  \ntime-consuming, and non-standardized nature presents significant obstacles to comprehensive evaluation and widespread adoption of LLMs in practice. This study reviews existing literature on human evaluation methodologies for LLMs in healthcare. We highlight a notable need for a standardized and consistent human evaluation approach. Our extensive literature search, adhering to the Preferred Reporting Items for Systematic Reviews and Meta-Analyses ( PRISMA) guidelines, includes publications from January 2018 to February 2024. The review examines the human evaluation of LLMs across various medical specialties, addressing factors such as evaluation dimensions, sample types and sizes, selection, and recruitment of evaluators, frameworks and metrics, evaluation process, and statistical analysis type. Drawing on the diverse evaluation strategies employed in these studies, we propose a comprehensive  \nand practical framework for human evaluation of LLMs: QUEST: Quality of Information, Understanding and Reasoning, Expression Style and Persona, Safety and Harm, and Trust and Confidence. This framework aims to improve the reliability, generalizability, and applicability of human evaluation of LLMs in different healthcare applications by defining clear evaluation dimensions and offering detailed guidelines.  \n1 . Introduction  \nDue to the remarkable ability to generate coherent responses to questions and requests, generative artificial intelligence (AI) , specifically large language models ( LLMs) such as proprietary LLMs (e.g. , GPT-4 1) and open-source LLMs (e.g. , LLaMA 2) , have rapidly gained popularity across various domains, including healthcare. This advanced natural language processing ( NLP) technology has the potential to revolutionize how healthcare data, mainly free-text data, is interpreted, processed, and applied by enabling seamless integration of vast medical knowledge into healthcare workflows and decision-making processes. For instance, LLMs can be leveraged for medical question answering 3 , providing healthcare professionalsand patients with evidence-bas","cbCaiifS5vV5cv8q","https://ap.wps.com/l/cbCaiifS5vV5cv8q","pdf",1727347,51,"English","# Abstract\n# 1. Introduction\n## Need for Human Evaluation\n## Limitations of Automated Metrics\n## Role of Human Evaluators","[{\"question\":\"Why is human evaluation important for large language models in healthcare?\",\"answer\":\"Because automated evaluations alone may not ensure the clinical utility, safety, reliability, and effectiveness required for healthcare deployment. Human evaluation addresses gaps that metrics cannot cover.\"},{\"question\":\"What scope and period does the literature review cover?\",\"answer\":\"The review follows PRISMA guidelines and includes publications from January 2018 to February 2024.\"},{\"question\":\"What is the QUEST framework used for?\",\"answer\":\"QUEST provides clear evaluation dimensions and detailed guidelines for human assessment of LLMs in healthcare, aiming to improve reliability, generalizability, and applicability across applications.\"}]","A Framework for Human Evaluation of Large Language Models in Healthcare - Derived from Literature Review | PDF",1791156715,129]