[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-426763-105":59,"doc-detail-426763-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","romeva-roman-urdu-embedding-preserving-vocabulary-adaptation","ROMEVA - Roman Urdu Embedding-preserving Vocabulary Adaptation","","Multilingual language models such as mBERT are widely used for low-resource NLP, but their adaptation to morphologically inconsistent Roman Urdu remains underexplored. Roman Urdu spelling variation causes severe sub-word fragmentation, averaging 1.50 sub-words per token. This work proposes ROMEVA, combining sub-word-average initialization and a PCA-guided anchor loss for stable embeddings during vocabulary expansion. Experiments on a 36,130-comment Roman Urdu corpus add 500 fragmented tokens to mBERT and compare naive tuning, sub-word-aware tuning, and ROMEVA.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/romeva-roman-urdu-embedding-preserving-vocabulary-adaptation/426763/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/romeva-roman-urdu-embedding-preserving-vocabulary-adaptation/426763.png","ImageObject",300,407,{"name":92,"@type":93},"dawugda","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-23","2026-09-29",true,{"@type":102,"interactionType":103,"userInteractionCount":34},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"Why is vocabulary adaptation difficult for Roman Urdu?","Question",{"text":112,"@type":113},"Roman Urdu has extensive spelling variation, so semantically identical words get different sub-word segmentations under mBERT, fragmenting evidence and weakening lexical consistency.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What is ROMEVA and how does it preserve embedding geometry?",{"text":117,"@type":113},"ROMEVA uses sub-word-average initialization together with a PCA-guided anchor loss to reduce embedding drift when expanding the vocabulary for Roman Urdu.",{"name":119,"@type":110,"acceptedAnswer":120},"What is the key finding about embedding stability and downstream performance?",{"text":121,"@type":113},"Improving embedding stability with ROMEVA does not guarantee better downstream sentiment classification; naive fine-tuning yields stronger sentiment results, indicating a tradeoff between representation preservation and task-specific adaptation.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},426763,1790981193,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":34,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":34,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":143,"read_time":144},3985747870947,"https://ap-avatar.wpscdn.com/davatar_3d24733baf745e90a7e4bdd5f77d97b2","ROMEVA: Geometry-Preserving Vocabulary Expansion for Roman Urdu Language Models  \narXiv :2606 .22478v1 [ cs .CL] 21 Jun 2026  \nMahnoor Khan 1 ,∗ , Afsheen Asif1 ,∗ , Milhan Afzal Khan2 , Seemab Latif1 , Mehwish Fatima 1 ,†  \n1 School of Electrical Engineering and Computer Science (SEECS),  \nNational University of Sciences and Technology (NUST),  \nIslamabad, Pakistan  \n{mkhan.msds25seecs, aasif.msds25seecs, seemab.latif, [mehwish.fatima}@seecs.edu.pk](mehwish.fatima}@seecs.edu.pk)  \n2Department of Computer Science,  \nUniversity of Agriculture Faisalabad,  \nFaisalabad, Pakistan  \n[milhankhan@uaf.edu.pk](milhankhan@uaf.edu.pk)  \n∗Equal contribution  \n†Corresponding [author: mehwish.fatima@seecs.edu.pk](author: mehwish.fatima@seecs.edu.pk)  \nAbstract—Multilingual Language Models like mBERT are widely used for low-resource NLP, yet their adaptation to morphologically inconsistent languages such as Roman Urdu remains underexplored. Roman Urdu spelling variation causes severe sub-word fragmentation, averaging 1.50 sub-words per token. We propose ROMEVA (Roman Urdu Embedding-preserving Vocabulary Adaptation), which combines sub-word-average initialization and a PCA-guided anchor loss to stabilize embeddings during vocabulary expansion. Using a 36,130-comment Roman Urdu corpus, we add 500 highly fragmented tokens to mBERT and compare naive fine-tuning, sub-word-aware fine-tuning, and ROMEVA. While ROMEVA most effectively preserves thepretrained embedding space, naive fine-tuning achieves the strongest downstream sentiment classification performance. These findings reveal a disconnect between embedding stability and downstream performance, suggesting that stronger adaptation maybe preferable to strict embedding preservation in morphologically inconsistent languages.  \nIndex Terms—Roman Urdu, Vocabulary Adaptation, Multilingual BERT, Embedding Stability, Representation Preservation, Low-Resource Languages, Domain-Adaptive Pretraining.  \nI. INTRODUCTION  \nMultilingual pretrained language models such as mBERT [1] have become pivotal to low-resource NLP because they transfer knowledge across languages through shared subword representations. These models perform well when the tokenizer aligns with the target language, but struggles with orthographically inconsistent languages. Table I illustrates the challenge. Semantically identical Roman Urdu words often appear in multiple spellings, each producing different sub-word segmentation under mBERT. As a result, the model distributes evidence for the same concept across multiple fragmented representations, reducing vocabulary coverage and weakening lexical consistency. Such variation fragments semantically related words into multiple sub-words and weakens the quality of representations. Roman Urdu is the dominant language of digital communication for millions of users across Pakistan and South Asia, particularly on social media, messaging platforms,  \nTABLE I  \nEXAMPLES OF ROMAN URDU SPELLING VARIATION AND MBERT TOKENIZATION .  \n\n| Meaning | Roman Urdu Forms | mBERT Tokens |\n| --- | --- | --- |\n| What | kya, kia, kiya | ky + \\#\\#a, ki + \\#\\#a, ki + \\#\\#ya |\n| Very | bohot, bohat | bo + \\#\\#hot, bo + \\#\\#hat |\n| Need | chahiye | cha + \\#\\#hi + \\#\\#ye |\n| Sometimes | kabhi, kbhi | ka + \\#\\#bhi, kb + \\#\\#hi |\n\nand online forums[2] . Its effective modeling is therefore important for downstream applications such as sentiment analysis, hate speech detection, information retrieval, and conversational AI.  \nA common response to vocabulary mismatch is domainadaptive pretraining (DAPT) [3] and vocabulary expansion, where domain-specific tokens are added to a pretrained model and the model is further adapted to target-domain data. These approaches improve vocabulary coverage and reduce fragmentation without requiring pretraining from scratch. However, multilingual tokenizers such as BPE [4] and SentencePiece [5] assume relatively stable surface forms, an assumption often violated in Roman Urdu due to extens","cbCaisXdOdlMtl8c","https://ap.wps.com/l/cbCaisXdOdlMtl8c","pdf",472401,"English","# Introduction\n## Problem: spelling variation and token fragmentation\n## Domain-adaptive pretraining and vocabulary expansion\n## Gap: embedding drift vs downstream performance\n# Related Work","[{\"question\":\"Why is vocabulary adaptation difficult for Roman Urdu?\",\"answer\":\"Roman Urdu has extensive spelling variation, so semantically identical words get different sub-word segmentations under mBERT, fragmenting evidence and weakening lexical consistency.\"},{\"question\":\"What is ROMEVA and how does it preserve embedding geometry?\",\"answer\":\"ROMEVA uses sub-word-average initialization together with a PCA-guided anchor loss to reduce embedding drift when expanding the vocabulary for Roman Urdu.\"},{\"question\":\"What is the key finding about embedding stability and downstream performance?\",\"answer\":\"Improving embedding stability with ROMEVA does not guarantee better downstream sentiment classification; naive fine-tuning yields stronger sentiment results, indicating a tradeoff between representation preservation and task-specific adaptation.\"}]","ROMEVA - Roman Urdu Embedding-preserving Vocabulary Adaptation | PDF",1790641718,18]