[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-1-en-105":3,"doc-seo-248237-105":53,"doc-detail-248237-en":127},{"code":4,"msg":5,"data":6},0,"success",[7,14,19,24,29,34,39,44,49],{"id":8,"doc_module":9,"doc_module_name":10,"category_name":11,"show_sort_weight":12,"slug":13},11,1,"Template","Presentations",90,"presentations",{"id":15,"doc_module":9,"doc_module_name":10,"category_name":16,"show_sort_weight":17,"slug":18},12,"Resumes",80,"resumes",{"id":20,"doc_module":9,"doc_module_name":10,"category_name":21,"show_sort_weight":22,"slug":23},14,"Invoices",70,"invoices",{"id":25,"doc_module":9,"doc_module_name":10,"category_name":26,"show_sort_weight":27,"slug":28},15,"Posters",60,"posters",{"id":30,"doc_module":9,"doc_module_name":10,"category_name":31,"show_sort_weight":32,"slug":33},16,"Social Media",50,"social-media",{"id":35,"doc_module":9,"doc_module_name":10,"category_name":36,"show_sort_weight":37,"slug":38},17,"Forms",40,"forms",{"id":40,"doc_module":9,"doc_module_name":10,"category_name":41,"show_sort_weight":42,"slug":43},18,"Letters",30,"letters",{"id":45,"doc_module":9,"doc_module_name":10,"category_name":46,"show_sort_weight":47,"slug":48},21,"Paper Templates",5,"papers-templates",{"id":50,"doc_module":9,"doc_module_name":10,"category_name":51,"show_sort_weight":4,"slug":52},158,"General","general-158",{"code":4,"msg":54,"data":55},"ok",{"site_id":56,"language":57,"slug":58,"title":59,"keywords":60,"description":61,"schema_data":62,"social_meta":120,"head_meta":122,"extra_data":124,"updated_unix":126},105,"en","orbit-open-recommendation-benchmark-for-reproducible-research-with-hidden-tests","ORBIT-Open Recommendation Benchmark for Reproducible Research with Hidden Tests","","Recommender systems drive daily experiences for billions of users, yet research progress is constrained by datasets that fail to reflect realistic user behavior and by evaluation setups that vary across splits, candidate pools, and metrics. This paper presents ORBIT, a unified open benchmark enabling reproducible and consistent evaluation for recommendation models. ORBIT standardizes public datasets with transparent settings and releases a new webpage recommendation task, ClueWeb-Reco, as hidden-test data derived from consented browsing histories with strong privacy safeguards.",{"@graph":63,"@context":119},[64,80,102],{"@type":65,"itemListElement":66},"BreadcrumbList",[67,71,74,77],{"item":68,"name":69,"@type":70,"position":9},"https://docshare.wps.com","Home","ListItem",{"item":72,"name":10,"@type":70,"position":73},"https://docshare.wps.com/template/",2,{"item":75,"name":11,"@type":70,"position":76},"https://docshare.wps.com/template/presentations/",3,{"item":78,"name":59,"@type":70,"position":79},"https://docshare.wps.com/template/orbit-open-recommendation-benchmark-for-reproducible-research-with-hidden-tests/248237/",4,{"url":78,"name":59,"@type":81,"image":82,"author":87,"headline":59,"publisher":90,"fileFormat":93,"inLanguage":57,"description":61,"dateModified":94,"datePublished":95,"encodingFormat":93,"isAccessibleForFree":96,"interactionStatistic":97},"DigitalDocument",{"url":83,"@type":84,"width":85,"height":86},"https://docshare.wps.com/thumbnails/orbit-open-recommendation-benchmark-for-reproducible-research-with-hidden-tests/248237.png","ImageObject",442,249,{"name":88,"@type":89},"Skyler","Person",{"url":68,"name":91,"@type":92},"DocShare","Organization","application/pdf","2026-10-28","2026-09-12",true,{"@type":98,"interactionType":99,"userInteractionCount":101},"InteractionCounter",{"@type":100},"ViewAction",7,{"@type":103,"mainEntity":104},"FAQPage",[105,111,115],{"name":106,"@type":107,"acceptedAnswer":108},"What problem does ORBIT target in recommender-systems research?","Question",{"text":109,"@type":110},"ORBIT addresses mismatches between existing datasets and real user behaviors, and inconsistent evaluation settings that make results hard to reproduce and compare fairly.","Answer",{"name":112,"@type":107,"acceptedAnswer":113},"What are ORBIT’s main components?",{"text":114,"@type":110},"ORBIT provides reproducible evaluations across 12 representative recommendation models on public datasets, and introduces a new webpage recommendation task, ClueWeb-Reco, as a hidden test set.",{"name":116,"@type":107,"acceptedAnswer":117},"How is ClueWeb-Reco constructed and used?",{"text":118,"@type":110},"ClueWeb-Reco is a synthetic dataset mapped from real, user-consented browsing sequences with privacy safeguards, and it is reserved as the hidden-test portion of ORBIT to assess generalization.","https://schema.org",{"og:url":78,"og:type":121,"og:title":59,"og:site_name":91,"og:description":61},"article",{"robots":123,"canonical":78},"index,follow",{"doc_id":125,"site_id":56},248237,1791739910,{"code":4,"msg":5,"data":128},{"doc_id":125,"user_id":129,"nickname":88,"user_avatar":130,"doc_module":9,"category_id":8,"category_name":11,"doc_title":59,"doc_description":61,"doc_content":131,"file_id":132,"file_url":133,"file_type":134,"file_size":135,"view_count":101,"is_deleted":4,"is_public":9,"is_downloadable":9,"audit_status":9,"page_count":136,"language":137,"language_code":57,"site_id":56,"html_lang":57,"table_of_contents":138,"faqs":139,"seo_title":140,"seo_description":61,"update_tm":141,"read_time":142},2336464648746,"https://ap-avatar.wpscdn.com/davatar_276721f389ce27ea32af1340a28f341c","arXiv :2510 .26095v 1 [ cs .IR] 30 Oct 2025  \nORBIT-Open Recommendation Benchmark for Reproducible Research with Hidden Tests  \nJingyuan He 1 Jiongnan Liu 1 Vishan Vishesh Oberoi 1 ∗ Bolin Wu 1 ∗ Mahima Jagadeesh Patel 1 ∗ Kangrui Mao 1 ∗ Chuning Shi 1 ∗  \nI-Ta Lee2 Arnold Overwijk2 Chenyan Xiong 1  \n1Language Technologies Institute, Carnegie Mellon University 2Meta  \n{jingyuah, jiongnal, voberoi, [bolinw}@andrew.cmu.edu](bolinw}@andrew.cmu.edu)[ ](bolinw}@andrew.cmu.edu){mjagadee, karrym, [chunings}@andrew.cmu.edu](chunings}@andrew.cmu.edu)[ ](chunings}@andrew.cmu.edu){arnoldov, [italee}@meta.com](italee}@meta.com) [cx@cs.cmu.edu](cx@cs.cmu.edu)  \nAbstract  \nRecommender systems are among the most impactful AI applications, interacting with billions of users every day, guiding them to relevant products, services, or information tailored to their preferences. However, the research and development of recommender systems are hindered by existing datasets that fail to capture realistic user behaviors and inconsistent evaluation settings that lead to ambiguous conclusions. This paper introduces the Open Recommendation Benchmark for Reproducible Research with HIdden Tests (ORBIT), a unified benchmark for consistent and realistic evaluation of recommendation models. ORBIT offers a standardized evaluation framework of public datasets with reproducible splits and transparent settings for its public leaderboard. Additionally, ORBIT introducesa new webpage recommendation task, ClueWeb-Reco, featuring web browsing sequences from 87 million public, high-quality webpages. ClueWeb-Reco is a synthetic dataset derived from real, user-consented, and privacy-guaranteed browsing data. It aligns with modern recommendation scenarios and is reserved asthe hidden test part of our leaderboard to challenge recommendation models’generalization ability. ORBIT measures 12 representative recommendation models on its public benchmark and introduces a prompted LLM baseline on the ClueWeb-Reco hidden test. Our benchmark results reflect general improvements of recommender systems on the public datasets, with variable individual performances. The results on the hidden test reveal the limitations of existing approaches in large-scale webpage recommendation and highlight the potential for improvements with LLM integrations. ORBIT benchmark, leaderboard, and codebase are available at [https://www.open-reco-bench.ai](https://www.open-reco-bench.ai).  \n1 Introduction  \nRecommender systems are among the most pervasive and influential AI applications today, enhancing user experience by delivering customized suggestions and reducing information overload from massive sources. Additionally, recommender systems are major revenue drivers for digital services and business platforms from e-commerce to social media applications. As such, recommender systems have become indispensable to both users and service providers and continue to attract significant attention from academia [1, 2, 3, 4] .  \nHowever, the progress of recommendation models in real-world settings often deviates significantly from that measured using publicly available research datasets. Many datasets rely on crawled reviews  \n∗Equal contribution.  \n39th Conference on Neural Information Processing Systems (NeurIPS 2025) Track on Datasets and Benchmarks.  \nand comments to approximate user action sequences, which differ substantially from other user interaction sequences in real world recommender systems (e.g., browsing and purchasing) . Some are constructed without explicit user consent, raising ethical and legal concerns [5] . Even worse, the evaluation setups across recommendation research vary greatly in many perspectives: data splits, inference-time candidate pool, and metrics. As reported in many previous studies [6, 7, 8], these discrepancies make it difficult to reproduce and compare results fairly across studies, leading to ambiguity of recommender system findings and slowing the advancement in this fiel","cbCaiv97gSMbLelF","https://ap.wps.com/l/cbCaiv97gSMbLelF","pdf",894849,29,"English","# Abstract\n# 1 Introduction","[{\"question\":\"What problem does ORBIT target in recommender-systems research?\",\"answer\":\"ORBIT addresses mismatches between existing datasets and real user behaviors, and inconsistent evaluation settings that make results hard to reproduce and compare fairly.\"},{\"question\":\"What are ORBIT’s main components?\",\"answer\":\"ORBIT provides reproducible evaluations across 12 representative recommendation models on public datasets, and introduces a new webpage recommendation task, ClueWeb-Reco, as a hidden test set.\"},{\"question\":\"How is ClueWeb-Reco constructed and used?\",\"answer\":\"ClueWeb-Reco is a synthetic dataset mapped from real, user-consented browsing sequences with privacy safeguards, and it is reserved as the hidden-test portion of ORBIT to assess generalization.\"}]","ORBIT-Open Recommendation Benchmark for Reproducible Research with Hidden Tests | PDF",1789244995,10]