[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-201820-105":59,"doc-detail-201820-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","conq-continuous-quantile-treatment-effects-for-large-scale-online-controlled-experiments","CONQ - CONtinuous Quantile Treatment Effects for Large-Scale Online Controlled Experiments","","Online controlled experimentation (A/B testing) is widely used to quantify product and feature impacts, but most work targets average treatment effects while performance metrics require quantile treatment effects (QTE). QTE estimation is difficult because clustered samples induce dependence and scalability and bandwidth choices complicate density estimation. Prior approaches often report only fixed quantiles such as P50/P90. This paper introduces CONQ, a scalable non-parametric method producing continuous QTE with point-wise confidence intervals while avoiding density estimation.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/conq-continuous-quantile-treatment-effects-for-large-scale-online-controlled-experiments/201820/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/conq-continuous-quantile-treatment-effects-for-large-scale-online-controlled-experiments/201820.png","ImageObject",300,407,{"name":92,"@type":93},"Jacob","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-29","2026-09-04",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",11,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What problem does the paper address in online controlled experiments?","Question",{"text":113,"@type":114},"It addresses how to measure quantile treatment effects for performance metrics, where mean-based evaluation is often ill-advised and quantile estimation is challenging.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"Why is estimating quantile treatment effects difficult?",{"text":118,"@type":114},"QTE estimation is challenged by dependence from clustered samples, scalability constraints, and issues such as bandwidth/density choices that strongly affect variance through density terms.",{"name":120,"@type":111,"acceptedAnswer":121},"What is CONQ and what does it provide?",{"text":122,"@type":114},"CONQ is a method that uses Bag of Little Bootstrap and the Bahadur representation to construct point-wise asymptotically valid confidence intervals and p-values for continuous QTE, consistent with prior normal-approximation results.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},201820,1788539077,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":44,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":130,"read_time":144,"preview_page_count":4},962084931830,"https://ap-avatar.wpscdn.com/davatar_a8503ba1806abce46bf441b54a3ca4cd","CONQ: CONtinuous Quantile Treatment Effects for Large-Scale  \nOnline Controlled Experiments  \nWeinan Wang  \n[wwang@snapchat.com](wwang@snapchat.com)[ ](wwang@snapchat.com)Snap Inc.  \nSanta Monica, California, USA  \nXi Zhang  \n[xzhang2@snapchat.com](xzhang2@snapchat.com)[ ](xzhang2@snapchat.com)Snap Inc.  \nSanta Monica, California, USA  \narXiv :2010 . 15326v1 [ stat .ME] 29 Oct 2020  \nABSTRACT  \nIn many industry settings, online controlled experimentation (A/B test) has been broadly adopted as the gold standard to measure product or feature impacts. Most research has primarily focused on user engagement type metrics, specifically measuring treatment effects at mean (average treatment effects, ATE), and only a few have been focusing on performance metrics (e.g. latency), where treatment effects are measured at quantiles. Measuring quantile treatment effects (QTE) is challenging due to the myriad difficulties such as dependency introduced by clustered samples, scalability issues, density bandwidth choices, etc. In addition, previous literature has mainly focused on QTE at some pre-defined locations, such as P50 or P90, which doesn’t always convey the full picture. In this paper, we propose a novel scalable non-parametric solution, which can provide a continuous range of QTE with point-wise confidence intervals while circumventing the density estimation altogether. Numerical results show high consistency with traditional methods utilizing asymptotic normality. An end-to-end pipeline has been implemented at Snap Inc., providing daily insights on key performance metrics at a distributional level.  \nKEYWORDS  \ncontrolled experiments, quantile treatment effect, scalability, nonparametric estimation, Bahadur representation, continuous.  \nACM Reference Format:  \nWeinan Wang and Xi Zhang. 2020. CONQ: CONtinuous Quantile Treatment Effects for Large-Scale Online Controlled Experiments. In Proceedings of WSDM’21: ACM Symposium on Neural Gaze Detection (WSDM’21). ACM, New York, NY, USA, 9 pages. [https://doi.org/10.1145/1122445.1122456](https://doi.org/10.1145/1122445.1122456)  \n1 INTRODUCTION  \nLarge-scale online experiments [8, 9] are routinely conducted at various technology companies (Google, Netflix, Microsoft, LinkedIn, etc.) [2, 22, 28, 31, 32] to help gauge quantifiable impacts of potential feature and product launches. At Snap Inc., hundreds of experiments are conducted on a daily basis, aiming to improve user engagement or app-performance. To accurately measure these changes, hundreds and thousands of metrics are further pumped through our in house A/B platform where routine statistical tests  \nPermission to make digital or hard copies of all or part of this work for personal or classroom use is granted without fee provided that copies are not made or distributed for profit or commercial advantage and that copies bear this notice and the full citation on the first page. Copyrights for components of this work owned by others than ACM must be honored. Abstracting with credit is permitted. To copy otherwise, or republish, to post on servers or to redistribute to lists, requires prior specific permission and/or a fee. Request permissions [from permissions@acm.org](from permissions@acm.org).  \nWSDM’21, March 08–12, 2021, Jerusalem, Israel © 2020 Association for Computing Machinery.  \nACM ISBN 978-1-4503-XXXX-X/18/06. . . $15.00 [https://doi.org/10.1145/1122445.1122456](https://doi.org/10.1145/1122445.1122456)  \nare conducted to judge if the treatment effects are indeed statistically significant. These metrics can be roughly divided into two types: engagement vs. performance. For engagement type metrics, typically two sample 􀁃-tests are utilized to determine if the average treatment effects (ATE) are truly nonzero [14] . However for performance metrics (e.g., camera transcoding latency, page load latency, etc.), evaluating treatment effects at mean is typically ill-advised [23], and the industry standard is to measure QTE (quantile treat","cbCais9OLYOynq1n","https://ap.wps.com/l/cbCais9OLYOynq1n","pdf",926652,"English","# Introduction\n## Measuring engagement vs performance metrics\n## Challenges in estimating quantile treatment effects\n## Proposed method: CONQ\n## Confidence intervals and variance reduction","[{\"question\":\"What problem does the paper address in online controlled experiments?\",\"answer\":\"It addresses how to measure quantile treatment effects for performance metrics, where mean-based evaluation is often ill-advised and quantile estimation is challenging.\"},{\"question\":\"Why is estimating quantile treatment effects difficult?\",\"answer\":\"QTE estimation is challenged by dependence from clustered samples, scalability constraints, and issues such as bandwidth/density choices that strongly affect variance through density terms.\"},{\"question\":\"What is CONQ and what does it provide?\",\"answer\":\"CONQ is a method that uses Bag of Little Bootstrap and the Bahadur representation to construct point-wise asymptotically valid confidence intervals and p-values for continuous QTE, consistent with prior normal-approximation results.\"}]","CONQ - CONtinuous Quantile Treatment Effects for Large-Scale Online Controlled Experiments | PDF",23]