[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-153780-105":59,"doc-detail-153780-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","lid-language-identification-of-short-text-in-transliterated-space","LID - Language identification of short text in transliterated space","","Huge volumes of social media content on platforms such as Twitter and Facebook use short text written in transliterated Roman form. Accurate language identification is needed to retrieve and organize information from these queries. This work focuses on transliterated Indian languages and evaluates Naive Bayes, Logistic Regression, and Support Vector Machine. Results show SVM with 4-gram features performs best, enabling integration with an IR platform to detect the language of incoming queries.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/lid-language-identification-of-short-text-in-transliterated-space/153780/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/lid-language-identification-of-short-text-in-transliterated-space/153780.png","ImageObject",300,407,{"name":92,"@type":93},"Hazel","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-30","2026-08-28",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",12,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"Why is language identification important for transliterated short text on social media?","Question",{"text":113,"@type":114},"Because search and information access require knowing the language of short Romanized queries, which are common in platforms like Twitter and Facebook.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"What dataset was created for this study?",{"text":118,"@type":114},"A corpus of manually annotated bilingual tweets in Bengali, Gujarati, and Hindi, cleaned by removing punctuation, URLs, and digits.",{"name":120,"@type":111,"acceptedAnswer":121},"Which classifier and feature setting performed best?",{"text":122,"@type":114},"Support Vector Machine with 4-gram features provided the best language identification results for transliterated text.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},153780,1787882295,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":19,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":130,"read_time":52,"preview_page_count":4},137441390410,"https://ap-avatar.wpscdn.com/avatar/2000252f4ab5702993?_k=1776741390130283984","Language identification of short text in transliterated space.  \nHarsha Kokel  \nDA-IICT, Gandhinagar  \n[200901199@daiict.ac.in](200901199@daiict.ac.in)  \nAbstract—Huge amount of data available on social media services like twitter, Facebook etc. comprises of short text in transliterated space. For such transliterated text there is a need to identify the language in order to access information. Language Identification is a well studied problem. Various classifiers are used for language identification but it is not evident how well they function with short text in Romanized Indian languages. We generated an annotated collection of Tweets in Roman script for three Indian languages: Bengali, Gujarati and Hindi. Three classifiers: Naive Bayes, Logistic Regression and Support Vector Machine were compared for language identification of transliterated text. It was found that SVM performs the best with 4-gram features. Hence integrated SVM with the Terrier- 3.5 IR platform to detect language of the query provided.  \nI. INTRODUCTION  \nIndia is a multilingual and multiscript country. There are 23 officially listed Indian languages and 12 officially listed Indian scripts. Most of the Indian languages have a script of their own. For e.g. Hindi and Marathi use Devanagari Script, Gujarati uses Gujarati Script, Bangla and Assamese use Bengali Script, Kannada uses Kannada Script, Tamil uses Tamil Script, Telugu uses Telugu Script etc. There are multiple standard keyboards available for using Indian scripts like inscript keyboard, phonetic keyboard, unicode keyboard etc. But layman abstain from using such keyboards and rather use Transliteration. It has been observed that in most of the social media and discussion forums of Indian context, Indian languages have been written in Roman alphabets. It is evident from twitter and facebook that good amount of Hindi text is written in transliterated Roman script. Majority portion of the Indian song lyrics found online are in Roman Script and to search these lyrics people use search engine like Google or Bing. It becomes inevident for such search engines to identify the language of query to provide Cross Script information like lyrics of the Hindi song in Devanagari Script as well as Roman Script when query is in Roman Script. Query logs of search engines are full of transliterated texts and if only I can identify the language of these Transliterated text. In this paper I focus on the language identification of Indian languages transliterated in Roman script only.  \nLanguage identification in such a transliterated space is a problem as  \n1) Indian languages have large character set, which is mapped to character set of size 26 in Roman Script while transliteration. This mapping is nor one to one neither many to one. for e.g.,  \n“f” and “q” both are represented as sha.  \n“t” and “V” both are represented as ta.  \n“j” can be represented as ˜j or z.  \nlike “mmdAr” (Majumadara) can be written as “Majumder” or “mazumdar”.  \n2) This transliteration do not abide by any standard rule and hence it has many spell variations according to the corresponding sound of the alphabets in both languages. Vowels in Hindi can be mapped to single or multiple alphabets in Roman Script.  \nfor e.g.,  \n“l”(Bh˜ula) can be written as “bhul” or “bhool”.“{h” can be written as “hai” or “hain”.  \n3) Humans have tendency to remove redundancies. Like in SMS Language, online users also skip vowels and replace alphabets with numbers  \nfor e.g.,  \n“nhF”(Na˜h˜i)˜ is written as “nhi” instead of “nahii”,  \n“pFC􀀃”(Piche) is written as “pe6y” instead of “piche”.  \n4) This transliterated text is multilingual i.e., it is comprised of words from an Indian language and a few from English or sometimes from a third language.  \nfor e.g.  \nBengali: “shon besi deri hole Producer er office giye unedited film ta dekhe chole asbo”  \nGujarati: “Jab we met, movie na joie hoye to aaje jao, Must Watch!!!”  \nHindi : “Hey, kahan hai?? kabse wait kar rahe hai, come soon.”  \nHe","cbCaiuwd6zqOpSfX","https://ap.wps.com/l/cbCaiuwd6zqOpSfX","pdf",171587,"English","# Introduction\n## Problem background\n# Data\n## Twitter corpus construction\n## Cleaning and preprocessing\n# Method and Models\n## Classifiers compared\n## SVM with 4-gram features\n# System Integration\n## Terrier-3.5 IR platform language detection","[{\"question\":\"Why is language identification important for transliterated short text on social media?\",\"answer\":\"Because search and information access require knowing the language of short Romanized queries, which are common in platforms like Twitter and Facebook.\"},{\"question\":\"What dataset was created for this study?\",\"answer\":\"A corpus of manually annotated bilingual tweets in Bengali, Gujarati, and Hindi, cleaned by removing punctuation, URLs, and digits.\"},{\"question\":\"Which classifier and feature setting performed best?\",\"answer\":\"Support Vector Machine with 4-gram features provided the best language identification results for transliterated text.\"}]","LID - Language identification of short text in transliterated space | PDF"]