[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-137007-105":59,"doc-detail-137007-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","shy-guys-a-light-weight-approach-to-detecting-robots-on-websites","Shy Guys - A Light-Weight Approach to Detecting Robots on Websites","","Automated bots generate roughly half of all web requests and increasingly spoof identities to evade detection and ignore robots.txt. Many existing countermeasures require heavy computation, raise costs, or harm user experience. The paper introduces a lightweight, passive detection method that analyzes user-agent strings and favicon-derived heuristics using only standard web server logs, with no client-side interaction. Evaluation on 4.6M+ requests shows 67.7% bot detection at a 3% false-positive rate, enabling a first-line defense.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/shy-guys-a-light-weight-approach-to-detecting-robots-on-websites/137007/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/shy-guys-a-light-weight-approach-to-detecting-robots-on-websites/137007.png","ImageObject",300,407,{"name":92,"@type":93},"Asher","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-06","2026-08-22",true,{"@type":102,"interactionType":103,"userInteractionCount":34},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"How does the proposed method detect robots on websites?","Question",{"text":112,"@type":113},"It combines user-agent string analysis with favicon-based heuristics, operating purely on standard web server logs without any client-side interaction.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What evidence shows the method is effective?",{"text":117,"@type":113},"The evaluation uses over 4.6 million requests with 54,945 unique user-agent strings and reports 67.7% bot detection with a 3% false-positive rate.",{"name":119,"@type":110,"acceptedAnswer":120},"How does this approach compare to state-of-the-art solutions?",{"text":121,"@type":113},"It is reported to outperform state-of-the-art methods with less than 20% improvement in detected effectiveness while keeping a low false-positive rate.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},137007,1787397704,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":34,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":52,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":129,"read_time":143},687197207639,"https://ap-avatar.wpscdn.com/davatar_a8503ba1806abce46bf441b54a3ca4cd","Shy Guys: A Light-Weight Approach to Detecting  \nRobots on Websites  \nRmi Van Boxem, Tom Barbette, Cristel Pelsser and Ramin Sadre  \nUCLouvain,  \nInstitute of Information and Communication Technologies, Electronics and Applied Mathematics, ICTEAM,  \nPle en ingnierie informatique, INGI  \nPlace Sainte Barbe 2, Louvain-la-Neuve, Belgium  \n[Email:](Email: {firstname.lastname}@uclouvain.be)[ {](Email: {firstname.lastname}@uclouvain.be)[firstname.lastname](Email: {firstname.lastname}@uclouvain.be)[}](Email: {firstname.lastname}@uclouvain.be)[@uclouvain.be](Email: {firstname.lastname}@uclouvain.be)  \narXiv :2603 .28546v2 [ cs .NI] 3 1 Mar 2026  \nAbstract—Automated bots now account for roughly half of all web requests, and an increasing number deliberately spoof their identity to either evade detection or to not respect robots .txt. Existing countermeasures are either resourceintensive (JavaScript challenges, CAPTCHAs), cost-prohibitive (commercial solutions), or degrade the user experience. This paper proposes a lightweight, passive approach to bot detection that combines user-agent string analysis with favicon-based heuristics, operating entirely on standard web server logs with no client-side interaction. We evaluate the method on over 4.6 million requests containing 54,945 unique user-agent strings collected from websites hosted all around the earth. Our approach detects 67.7% of bot traffic while maintaining a false-positive rate of 3%, outperforming state of the art (less than 20%). This method can serve as a first line of defense, routing only genuinely ambiguous requests to active challenges and preserving the experience of legitimate users.  \nI. INTRODUCTION  \nIn late 2021, a theory called “Dead Internet Theory” began to gain traction on the internet. This theory suggests that a significant portion of internet traffic is generated by bots rather than humans [1] . While often dismissed as a conspiracy theory, the latest annual report from Cloudflare indicates that bots (whether AI or non-AI) are responsible for half of requests to HTML pages, seven percentage points above human-generated traffic [2] . The same report also shows that traffic generated by AI crawlers increased to 15 times more than before.  \nIn April 2025, Wikimedia, the organization behind Wikipedia, published a blog post explaining its struggle to keep up with bots [3] . While traffic spikes are not uncommon when major events occur and while Wikimedia previously handled such surges without difficulty, it explained that baseline traffic had increased by 50% since January 2024 and that the organization has had greater difficulty serving content when major events (traffic spikes) occur. Further analysis showed that 65% of its most expensive traffic comes from bots. Due to Wikimedia’s infrastructure, expensive traffic is associated with less popular pages (less popular pages are not always cached and are requested from the main server) . Scrapers tend to read larger numbers of pages in bulk than humans, not only the most popular ones, thereby defeating caching strategies. One year later, the organization reported that it blocks or throttles approximately 25% of all automated requests and yet continues to face challenges as a new generation of crawlers  \nspoofs the identities of real web browsers and routes traffic through residential proxies to blend in with legitimate users [4] .  \nThis sudden increase in bot traffic can pose a serious threat to performance. It is not uncommon to observe content creators struggling to protect their websites [5], [6], forcing them to adopt CAPTCHA; migrate to hosting services that can detect bots and absorb the load of undetected bots; or employ aggressive measures that may block legitimate users (e.g., geoblocking, mandatory JavaScript challenges) [7] . One of the most common methods to identify bots is to examine their self-reported user-agent string. However, this method is not very reliable, as bots can easily spoof their user","cbCaifu1Ed1bh3nd","https://ap.wps.com/l/cbCaifu1Ed1bh3nd","pdf",355986,"English","# I. Introduction\n# II. Background and Related Work\n## A. Background\n# III. Proposed Approach\n# IV. Evaluation Results\n# V. Discussion and Results\n# VI. Ethical Considerations","[{\"question\":\"How does the proposed method detect robots on websites?\",\"answer\":\"It combines user-agent string analysis with favicon-based heuristics, operating purely on standard web server logs without any client-side interaction.\"},{\"question\":\"What evidence shows the method is effective?\",\"answer\":\"The evaluation uses over 4.6 million requests with 54,945 unique user-agent strings and reports 67.7% bot detection with a 3% false-positive rate.\"},{\"question\":\"How does this approach compare to state-of-the-art solutions?\",\"answer\":\"It is reported to outperform state-of-the-art methods with less than 20% improvement in detected effectiveness while keeping a low false-positive rate.\"}]","Shy Guys - A Light-Weight Approach to Detecting Robots on Websites | PDF",25]