[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-464090-105":59,"doc-detail-464090-en":134},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":127,"head_meta":129,"extra_data":131,"updated_unix":133},105,"en","gru-based-deep-multimodal-fusion-of-speech-and-head-imu-signals-in-mixed-reality-for-parkinsons-disease-detection","GRU-Based Deep Multimodal Fusion of Speech and Head-IMU Signals in Mixed Reality for Parkinson’s Disease Detection","","Parkinson’s disease (PD) alters both speech and movement, yet many automated assessments analyze these signals separately. This study tests whether combining voice with head motion improves discrimination between PD patients and healthy controls using synchronous acoustic and inertial measurements collected with a HoloLens 2 headset. Data from 165 participants (72 PD/93 HC) are evaluated with 5-fold stratified cross-validation. Voice-only performs strongly (AUC ≈0.865), inertial alone is near chance (AUC ≈0.497). Multimodal fusion yields modest, repeatable gains, with gated early-fusion best (AUC ≈0.875) and cross-attention fusion comparable (≈0.873).",{"@graph":69,"@context":126},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/gru-based-deep-multimodal-fusion-of-speech-and-head-imu-signals-in-mixed-reality-for-parkinsons-disease-detection/464090/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/gru-based-deep-multimodal-fusion-of-speech-and-head-imu-signals-in-mixed-reality-for-parkinsons-disease-detection/464090.png","ImageObject",300,407,{"name":92,"@type":93},"ahduag","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-03","2026-09-30",true,{"@type":102,"interactionType":103,"userInteractionCount":29},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118,122],{"name":109,"@type":110,"acceptedAnswer":111},"What signals are combined in this study for Parkinson’s detection?","Question",{"text":112,"@type":113},"The study combines speech (acoustic) signals with head inertial signals (Head-IMU), measured synchronously during mixed reality assessment.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How were the models evaluated between patients and healthy controls?",{"text":117,"@type":113},"Models were benchmarked using 5-fold stratified cross-validation, comparing single-modality approaches against fusion strategies.",{"name":119,"@type":110,"acceptedAnswer":120},"Which fusion strategy achieved the best discrimination performance?",{"text":121,"@type":113},"Gated early-fusion achieved the highest AUC (≈0.875), while cross-attention fusion was comparable (≈0.873).",{"name":123,"@type":110,"acceptedAnswer":124},"Why do the authors argue multimodal fusion helps beyond audio-only?",{"text":125,"@type":113},"Improvements were task-dependent: when tasks embed movement, complementary inertial information adds value, and motion acts as a conditional improvement factor rather than a sole predictor.","https://schema.org",{"og:url":83,"og:type":128,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":130,"canonical":83},"index,follow",{"doc_id":132,"site_id":62},464090,1791278647,{"code":4,"msg":5,"data":135},{"doc_id":132,"user_id":136,"nickname":92,"user_avatar":137,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":138,"file_id":139,"file_url":140,"file_type":141,"file_size":142,"view_count":29,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":143,"language":144,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":145,"faqs":146,"seo_title":147,"seo_description":67,"update_tm":148,"read_time":149,"preview_page_count":19,"preview_urls":150},3985747870414,"https://ap-avatar.wpscdn.com/davatar_9964176cb1d06d4a9deccf72a44ae3dc","Article  \nGRU-Based Deep Multimodal Fusion of Speech and Head-IMU Signals in Mixed Reality for Parkinson’s Disease Detection  \nDaria Hemmerling 1, *, Milosz Dudek 1, Justyna Krzywdziak 1, Magda Z˙bik 1, Wojciech Szecowka 1, Mateusz Daniol 1, Marek Wodzinski 1, Monika Rudzinska-Bar 2 and Magdalena Wojcik-Pedziwiatr 2  \nAcademic Editor: Rosario Schiano Lo Moriello  \nReceived: 22 October 2025  \nRevised: 9 December 2025  \nAccepted: 19 December 2025  \nPublished: 1 January 2026  \nCopyright: © 2026 by the authors. Licensee MDPI, Basel, Switzerland. This article is an open access article distributed under the terms and conditions of the Creative Commons Attribution (CC BY) license.  \n1 Department of Measurement and Electronics, AGH University of Krakow, 30-059 Krakow, Poland  \n2 Department of Neurology, Andrzej Frycz Modrzewski Krakow University, 30-705 Krakow, Poland  \n* [Correspondence: hemmer@agh.edu.pl](Correspondence: hemmer@agh.edu.pl)  \nAbstract  \nParkinson’s disease (PD) alters both speech and movement, yet most automated assessments still treat these signals separately. We examined whether combining voice with head motion improves discrimination between patients and healthy controls (HC) . Synchronous measurements of acoustic and inertial signals were collected using a HoloLens 2 headset. Data were obtained from 165 participants (72 PD/93 HC), following a standardized mixed-reality (MR) protocol. We benchmarked single-modality models against fusion strategies under 5-fold stratified cross-validation. Voice alone was robust (pooled AUC ≈ 0.865), while the inertial channel alone was near chance (AUC ≈ 0.497) . Fusion provided a modest but repeatable improvement: gated early-fusion achieved the highest AUC (≈0.875), cross-attention fusion was comparable (≈0.873) . Gains were task-dependent. While speech-dominated tasks were already well captured by audio, tasks that embed movement benefited from complementary inertial data. Proposed MR capture proved feasible within a single session and showed that motion acts as a conditional improvement factor rather than a sole predictor. The results outline a practical path to multimodal screening and monitoring for PD, preserving the reliability of acoustic biomarkers while integrating kinematic features when they matter.  \nKeywords: Parkinson’s disease; augmented reality (AR); speech analysis; voice biomarkers; inertial measurement units (IMU); head motion; multimodal data fusion; gyroscope and accelerometer signals; wearable sensing; remote digital assessment  \n1. Introduction  \nParkinson’s disease (PD) is a neurodegenerative disorder characterized by both motor and non-motor disturbances arising from dysfunction of basal ganglia–thalamo–cortical circuits. Deficits in movement initiation, scaling, and automatization give rise to the cardinal motor signs (bradykinesia, rigidity, tremor, postural instability) and to prominent axial manifestations such as reduced trunk–head rotation, impaired segmental coordination, and postural deformities [1,2] . Speech is frequently affected: hypokinetic dysarthria occurs in approximately 80–90% of patients and is characterized by hypophonia (reduced loudness), monopitch and monoloudness, flattened prosody, short rushes of speech, and imprecise articulation [2,3] . Mechanistically, these abnormalities reflect a combination of bradykinesia, peripheral rigidity, impaired amplitude scaling, and disrupted sensory control—including underestimation of self-produced loudness [1] . Endoscopic and imaging studies have  \ndocumented glottal insufficiency and bowing or atrophy of the vocal folds, while aerodynamic analyses show reduced subglottal pressure and altered loudness strategies (Lombard effect), pointing to combined respiratory–laryngeal dysfunction [4,5] .  \nMotor scaling deficits in PD extend across both orofacial and axial effectors. Axial symptoms include pronounced rigidity, reduced range of trunk–head rotation, en bloc coupling of head and trunk during","cbCaivWw6kEknWl9","https://ap.wps.com/l/cbCaivWw6kEknWl9","pdf",7850810,26,"English","# Abstract\n# Introduction","[{\"question\":\"What signals are combined in this study for Parkinson’s detection?\",\"answer\":\"The study combines speech (acoustic) signals with head inertial signals (Head-IMU), measured synchronously during mixed reality assessment.\"},{\"question\":\"How were the models evaluated between patients and healthy controls?\",\"answer\":\"Models were benchmarked using 5-fold stratified cross-validation, comparing single-modality approaches against fusion strategies.\"},{\"question\":\"Which fusion strategy achieved the best discrimination performance?\",\"answer\":\"Gated early-fusion achieved the highest AUC (≈0.875), while cross-attention fusion was comparable (≈0.873).\"},{\"question\":\"Why do the authors argue multimodal fusion helps beyond audio-only?\",\"answer\":\"Improvements were task-dependent: when tasks embed movement, complementary inertial information adds value, and motion acts as a conditional improvement factor rather than a sole predictor.\"}]","GRU-Based Deep Multimodal Fusion of Speech and Head-IMU Signals in Mixed Reality for Parkinson’s Disease Detection | PDF",1790767658,66,[151,155,159,163],{"page_no":8,"url":152,"width":153,"height":154},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/cd8f24f3864b39666be97eee173449f9.webp",983,1390,{"page_no":14,"url":156,"width":157,"height":158},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/1689645690ec2b7e897e3e678ee7716f.webp",982,1388,{"page_no":81,"url":160,"width":161,"height":162},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/f93512c63db20735fcb29885bd391650.webp",981,693,{"page_no":19,"url":164,"width":165,"height":166},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/bc413eb9f1cb44b325fae1553fe40ce7.webp",980,692]