[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-273690-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-273690-en":131},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","healmed-multilingual-evaluation-of-large-language-models-in-medicine-expert-reviewed-medical-benchmark","HealMed - Multilingual Evaluation of Large Language Models in Medicine - Expert-reviewed medical benchmark","","HealMed is an expert-reviewed benchmark designed for multilingual evaluation of large language models in medicine. It provides 1,000 examples per language across nine languages, sourced from nine datasets and covering MCQA, NLI, and open-ended QA. Built over two years by physicians and medical experts, each translation is assessed and revised by bilingual reviewers in English and the target language. Results show performance drops strongest in low-resource languages, while translation quality and expert revision can raise or lower measured scores.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/healmed-multilingual-evaluation-of-large-language-models-in-medicine-expert-reviewed-medical-benchmark/273690/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/healmed-multilingual-evaluation-of-large-language-models-in-medicine-expert-reviewed-medical-benchmark/273690.png","ImageObject",300,407,{"name":42,"@type":43},"River Wang","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-10-27","2026-09-15",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",17,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What does HealMed benchmark evaluate across languages in medicine?","Question",{"text":63,"@type":64},"HealMed evaluates multilingual medical understanding using three task formats: MCQA, NLI, and open-ended QA across nine languages.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How are the benchmark translations reviewed to ensure evaluation quality?",{"text":68,"@type":64},"Each translation undergoes a structured two-stage review and revision by two medical experts fluent in English and the corresponding target language.",{"name":70,"@type":61,"acceptedAnswer":71},"What patterns does HealMed find in performance across high- vs low-resource languages?",{"text":72,"@type":64},"Performance declines most in low-resource languages, with the language gap varying by model and showing task-specific losses concentrated in certain languages.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},273690,1789472503,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,111,115,120,123,127],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":108,"show_sort_weight":109,"slug":110},7,"Healthcare",40,"healthcare",{"id":112,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":113,"slug":114},8,30,"research-report",{"id":116,"doc_module":4,"doc_module_name":25,"category_name":117,"show_sort_weight":118,"slug":119},9,"Religion & Spirituality",20,"religion-spirituality",{"id":118,"doc_module":4,"doc_module_name":25,"category_name":121,"show_sort_weight":118,"slug":122},"World Cup","world-cup",{"id":124,"doc_module":4,"doc_module_name":25,"category_name":125,"show_sort_weight":124,"slug":126},10,"Lifestyle","lifestyle",{"id":128,"doc_module":4,"doc_module_name":25,"category_name":129,"show_sort_weight":97,"slug":130},19,"General","general",{"code":4,"msg":82,"data":132},{"doc_id":79,"user_id":133,"nickname":42,"user_avatar":134,"doc_module":4,"category_id":112,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":140,"language":141,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":12,"update_tm":80,"read_time":145,"preview_page_count":4},1099514067438,"https://ap-avatar.wpscdn.com/avatar/100002539ee87300030?x-image-process=image/resize,m_fixed,w_180,h_180&k=1780474512215547542","HealMed: Multilingual Evaluation of Large Language Models in Medicine  \nHealMed Research Team  \nSee the Contributions section for the complete list of contributors and affiliations.  \n HealMed  \narXiv : 2608 . 1998 1v 1 [cs .CL] 20 Aug 2026  \nAbstract  \nWe present HealMed, an expert-reviewed benchmark for multilingual evaluation of large language models in medicine. HealMed contains 1,000 examples in each of nine languages, drawn from nine datasets and covering three task formats: MCQA, NLI and open-ended QA. The benchmark was developed over two years by 23 physicians and medical experts based across nine countries and regions. Each translation was evaluated and revised by two experts fluent in English and the corresponding target language. On HealMed, performance declined most in low-resource languages, although the size of the gap varied markedly across languages and models. The strongest proprietary models were the most stable across languages, whereas many open-source and medically specialized models showed larger and less consistent gaps. Medical specialization alone did not ensure multilingual robustness. Furthermore, expert revision could either raise or lower measured performance, indicating that translation quality materially affects cross-language evaluation results.  \n1 Introduction  \nLarge language models (LLMs) have achieved strong performance on questions from medical licensing examinations and can generate long-form medical responses that receive favourable clinical ratings (Singhal et al., 2023, 2025) . Yet evidence for these capabilities remains centred on English (Ahuja et al., 2023 ; Wu et al., 2025 ; Xuan et al., 2025) . Performance on English benchmarks does not establish whether a model can interpret medical concepts or communicate them reliably in other languages, especially those underrepresented in training data and existing evaluations (Chenet al., 2025b ; Singhal et al., 2025) . Multilingual assessment is therefore needed to determine where medical competence transfers across languages and where it fails.  \nSeveral multilingual medical benchmarks now cover question answering and clinical language understanding (Qiu et al., 2024 ; Alonso et al., 2024 ; Wu et al., 2026) . Many, however, are built by translating English test sets. Translation can alter medical terminology, syntax and contextual meaning. These changes can affect model scores and rankings (Artetxe et al., 2020 ; Singh et al., 2025) . A lower score in a target language may therefore reflect weaker model capability, translation error or both. Without expert review, these sources of error cannot be separated.  \nIn this paper, we present HealMed: Humanverified Evaluation Across Languages for Medical AI, an expert-reviewed medical benchmark spanning nine languages, nine source datasets and three task families: multiple-choice question answering (MCQA), natural language inference (NLI) and open-ended question answering (QA) . 23 physicians and medical experts contributed to its construction and expert review. Every targetlanguage instance underwent a structured twostage review by two medical experts fluent in English and the corresponding target language. We evaluated 14 LLMs on MCQA and NLI and a subset of ten models on open-ended QA. The panel comprised five proprietary models (GPT-5.4, o4-mini, Gemini-3-Flash, Claude-Sonnet-5 and Claude-Opus-4.8) (OpenAI, 2026, 2025 ; Google DeepMind, 2025 ; Anthropic, 2026b,a), six generalpurpose open-source model configurations from the DeepSeek (DeepSeek-AI, 2024), Qwen (Yang et al., 2024, 2025), LLaMA (Grattafiori et al., 2024) and Gemma (Gemma Team et al., 2025) families, and three medically specialized models: HuatuoGPT-o1 (Chen et al., 2025a), MedGemma (Sellergren et al., 2025) and MediPhi (Corbeil et al., 2025) . Open-ended responses were assessed using a multilingual LLMas-judge protocol. The original machine-translated and final expert-reviewed versions were retained  \na. HealMed benchmark scope an","cbCaib8hFuBrEm3Y","https://ap.wps.com/l/cbCaib8hFuBrEm3Y","pdf",2192830,38,"English","# Introduction\n## Benchmark motivation for multilingual medical AI\n## Translation risks and need for expert separation of error sources\n## HealMed overview and scope\n## Benchmark construction pipeline and expert review\n## Evaluation setup across model types and languages\n## Findings on language gaps and task performance","[{\"question\":\"What does HealMed benchmark evaluate across languages in medicine?\",\"answer\":\"HealMed evaluates multilingual medical understanding using three task formats: MCQA, NLI, and open-ended QA across nine languages.\"},{\"question\":\"How are the benchmark translations reviewed to ensure evaluation quality?\",\"answer\":\"Each translation undergoes a structured two-stage review and revision by two medical experts fluent in English and the corresponding target language.\"},{\"question\":\"What patterns does HealMed find in performance across high- vs low-resource languages?\",\"answer\":\"Performance declines most in low-resource languages, with the language gap varying by model and showing task-specific losses concentrated in certain languages.\"}]","HealMed - Multilingual Evaluation of Large Language Models in Medicine - Expert-reviewed medical benchmark | PDF",96]