[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-317024-105":59,"doc-detail-317024-en":123},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":116,"head_meta":118,"extra_data":120,"updated_unix":122},105,"en","medqa-cs-objective-structured-clinical-examination-osce-style-benchmark-for-evaluating-llm-clinical-skills","MedQA-CS: Objective Structured Clinical Examination (OSCE)-Style Benchmark for Evaluating LLM Clinical Skills","","Artificial intelligence and large language models in healthcare require advanced clinical skills, yet existing benchmarks do not evaluate these skills comprehensively. The paper introduces MedQA-CS, an AI-SCE framework inspired by medical education’s Objective Structured Clinical Examinations (OSCEs). It evaluates LLMs using two instruction-following tasks—LLM-as-medical-student and LLM-as-CS-examiner—designed to mirror clinical scenarios. The work releases data and expert annotations and demonstrates stronger challenge than multiple-choice QA benchmarks, enabling broader assessment of clinical capability for open- and closed-source models.",{"@graph":69,"@context":115},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/medqa-cs-objective-structured-clinical-examination-osce-style-benchmark-for-evaluating-llm-clinical-skills/317024/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/medqa-cs-objective-structured-clinical-examination-osce-style-benchmark-for-evaluating-llm-clinical-skills/317024.png","ImageObject",300,407,{"name":92,"@type":93},"Fahsai","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-10","2026-09-20",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",17,{"@type":107,"mainEntity":108},"FAQPage",[109],{"name":110,"@type":111,"acceptedAnswer":112},"Why is USMLE Step 2 CS relevant to the MedQA-CS design?","Question",{"text":113,"@type":114},"USMLE Step 2 CS provides an OSCE-style workflow with phased patient interaction, feedback, and criterion-based scoring, which the paper adapts into an AI-SCE setting for clinical skills benchmarking.","Answer","https://schema.org",{"og:url":83,"og:type":117,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":119,"canonical":83},"index,follow",{"doc_id":121,"site_id":62},317024,1790335174,{"code":4,"msg":5,"data":124},{"doc_id":121,"user_id":125,"nickname":92,"user_avatar":126,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":127,"file_id":128,"file_url":129,"file_type":130,"file_size":131,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":132,"language":133,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":134,"faqs":135,"seo_title":136,"seo_description":67,"update_tm":137,"read_time":138,"preview_page_count":4},549768702563,"https://ap-avatar.wpscdn.com/avatar/8000c4aa63b76e948b?x-image-process=image/resize,m_fixed,w_180,h_180&k=1786536092046926083","MedQA-CS: Objective Structured Clinical Examination (OSCE)-Style Benchmark for Evaluating LLM Clinical Skills  \nZonghai Yao 1 , Zihao Zhang 2 , Chaolong Tang 1 , Xingyu Bian 1  \nYouxia Zhao 1 , Zhichao Yang 1 , Junda Wang 1 , Huixue Zhou 3 Won Seok Jang 4 , Feiyun Ouyang 4 , Hong Yu 1 ,4 ,5  \nUniversity of Massachusetts, Amherst 1 , Emory University2 , University of Minnesota3  \nUniversity of Massachusetts, Lowell4 , UMass Chan Medical School5  \n{zonghaiyao, chaolongtang, xingyubian, [youxiazhao}@umass.edu](youxiazhao}@umass.edu)  \nAbstract  \nArtificial intelligence (AI) and large language models (LLMs) in healthcare require advanced clinical skills (CS), yet current benchmarks fail to evaluate these comprehensively. We introduce MedQA-CS, an AI-SCE framework inspired by medical education’s Objective Structured Clinical Examinations (OSCEs), to address this gap. MedQA-CS evaluates LLMs through two instruction-following tasks—LLM-as-medical-student and LLM-asCS-examiner—designed to reflect real clinical scenarios. Our contributions include developing MedQA-CS, a comprehensive evaluation framework with publicly available data and expert annotations, and providing the quantitative and qualitative assessment of LLMs as reliable judges in CS evaluation. Our experiments show that MedQA-CS is a more challenging benchmark for evaluating clinical skills than traditional multiple-choice QA benchmarks (e.g., MedQA) . Combined with existing benchmarks, MedQA-CS enables a more comprehensive evaluation of LLMs’ clinical capabilities for both open-and closed-source LLMs 1 .  \n1 Introduction  \nArtificial intelligence (AI) and large language models (LLMs) are increasingly adopted in healthcare, resulting in many clinical NLP applications that require expert-level clinical skills such as diagnosis and clinical documentation (Achiam et al., 2023 ; McDuff et al., 2023 ; Tu et al., 2024 ; Yang et al., 2024) . Current clinical LLM benchmarks, such as MMLU-Med (Hendrycks et al., 2020), MedQAUS (Jin et al., 2021), MedMCQA (Pal et al., 2022), PubMedQA (Jin et al., 2019), JAMA Clinical Challenge, Medbullets (Chen et al., 2024), and AM-  \n1 Our data, prompts, codes, and annotations are public at GitHub [https://github.com/bio-nlp/MedQA-CS](https://github.com/bio-nlp/MedQA-CS)and Hugging Face Hub [https://huggingface.co/](https://huggingface.co/)[ ](https://huggingface.co/)datasets/bio-nlp-umass/MedQA-CS-Student and [https://huggingface.co/datasets/bio-nlp-umass/](https://huggingface.co/datasets/bio-nlp-umass/)[ ](https://huggingface.co/datasets/bio-nlp-umass/)MedQA-CS-Exam with CC-BY-NC-4 .0 License.  \nFigure 1: Miller’s pyramid of clinical competence matched with an appropriate level of assessment. Figure adapted from (Miller, 1990) .  \nBOSS (Gilson et al., 2023), mainly measure clinical knowledge through multiple-choice questions (MCQ) .  \nHowever, identifying robust clinical guidelines and what constitutes a successful interaction for healthcare LLMs will be crucial towards fulfilling the long-term goals of patients, providers, and other clinical stakeholders (Mehandru et al., 2024) . In medical education, there has been a shift from assessing students using standardized testing, which evaluates clinical knowledge through MCQs, to modern curricula, which increasingly use Objective Structured Clinical Examination (OSCE) (Zayyan, 2011 ; Harden et al., 2015) . As shown in Figure 1, Miller’s Pyramid (Miller, 1990) provides a comprehensive framework for evaluating the competence of medical students, from knowledge acquisition to real-world performance (Norcini, 2003 ; Albino et al., 2008) . Early medical exams have typically evaluated students on the \"knows\" and \"knowshow\" levels of Miller’s Pyramid, while OSCEs primarily evaluate students’ practical skills (e.g., the\"shows how\" level) in clinical settings, including patient examination, clinical history recording, effective communication, and handling unexpected situations. As depicted in Figure 2, the U","cbCaisK4qBdnBKE9","https://ap.wps.com/l/cbCaisK4qBdnBKE9","pdf",1200281,75,"English","# Abstract\n# Introduction\n## Clinical Skills Benchmarks and Gaps\n## OSCE and Competence Assessment Frameworks\n## USMLE Step 2 CS as a Structural Reference\n# MedQA-CS Framework and Tasks","[{\"question\":\"Why is USMLE Step 2 CS relevant to the MedQA-CS design?\",\"answer\":\"USMLE Step 2 CS provides an OSCE-style workflow with phased patient interaction, feedback, and criterion-based scoring, which the paper adapts into an AI-SCE setting for clinical skills benchmarking.\"}]","MedQA-CS: Objective Structured Clinical Examination (OSCE)-Style Benchmark for Evaluating LLM Clinical Skills | PDF",1789927778,189]