[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-202151-105":59,"doc-detail-202151-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","an-online-sequential-test-for-qualitative-treatment-effects-research-paper","An Online Sequential Test for Qualitative Treatment Effects - Research Paper","","Tech companies often rely on randomized online experiments and A/B testing, usually driven by average treatment effects, to compare a new product with an existing one. Detecting qualitative treatment effects is crucial when the new product outperforms the old only under specific circumstances. This paper introduces a powerful testing procedure for qualitative treatment effects using an online updating algorithm with adaptive randomization, sequential monitoring, and guaranteed type-I error control.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/an-online-sequential-test-for-qualitative-treatment-effects-research-paper/202151/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/an-online-sequential-test-for-qualitative-treatment-effects-research-paper/202151.png","ImageObject",300,407,{"name":92,"@type":93},"Mali","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-01","2026-09-04",true,{"@type":102,"interactionType":103,"userInteractionCount":29},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What problem does the paper address in online A/B testing?","Question",{"text":112,"@type":113},"It addresses how to detect qualitative treatment effects, where the new product performs significantly better only under certain conditions, not just on average.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does the proposed method control error in sequential online experiments?",{"text":117,"@type":113},"It uses an online updating algorithm with adaptive randomization, sequential monitoring, and guaranteed type-I error control based on a spending-function style approach.",{"name":119,"@type":110,"acceptedAnswer":120},"Why can’t earlier heterogeneous treatment effect tests be directly applied here?",{"text":121,"@type":113},"Heterogeneous treatment effects (HTE) and qualitative treatment effects (QTE) are related but fundamentally different hypotheses; cases exist where HTE holds while QTE does not.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},202151,1791119800,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":29,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":145,"preview_page_count":4},2336475104362,"https://ap-avatar.wpscdn.com/avatar/22000c4c46a41b752dd?x-image-process=image/resize,m_fixed,w_180,h_180&k=1786595829695023868","An Online Sequential Test for Qualitative Treatment E􀀋ects  \nChengchun Shi [c.shi7@lse.ac.uk](c.shi7@lse.ac.uk)  \nDepartment of Statistics, London School of Economics and Political Science  \nShikai Luo [sluo198912@163.com](sluo198912@163.com)  \nTecent PCG  \nHongtu Zhu [htzhu@email.unc.edu](htzhu@email.unc.edu)  \n[Department of Biostatistics](Department of Biostatistics), [University of North-Carolina](University of North-Carolina)  \nRui Song [rsong@ncsu.edu](rsong@ncsu.edu)  \n[Department of Statistics](Department of Statistics), [North-Carolina State University](North-Carolina State University)  \n[Editor:](Editor: Xiaotong Shen)[ Xiaotong Shen](Editor: Xiaotong Shen)  \nAbstract  \nTech companies (e.g., Google or Facebook) often use randomized online experiments and/or A/B testing primarily based on the average treatment e􀀋ects to compare their new product with an old one. However, it is also critically important to detect qualitative treatment e􀀋ects such that the new one may signi􀀌cantly outperform the existing one only under some speci􀀌c circumstances. The aim of this paper is to develop a powerful testing procedure toe􀀎ciently detect such qualitative treatment e􀀋ects. We propose a scalable online updating algorithm to implement our test procedure. It has three novelties including adaptive randomization, sequential monitoring, and online updating with guaranteed type-I error control. We also thoroughly examine the theoretical properties of our testing procedure including the limiting distribution of test statistics and the justi􀀌cation of an e􀀎cient bootstrap method. Extensive empirical studies are conducted to examine the 􀀌nite sample performance of our test procedure.  \nKeywords: A/B testing; Qualitative treatment e􀀋ects; Sequential monitoring; Adaptive randomization; Online updating.  \n1. Introduction  \nTech companies use randomized online experiments, or A/B testing to compare their new product with a well-established one. Most works in the literature focus on the average treatment e􀀋ects (ATE) between the new and existing products (see Kharitonov et al. , 2015; Johari et al., 2015, 2017; Yang et al., 2017; Ju et al., 2019, and the references therein) . In addition to ATE, sometimes we are interested in locating the subgroup (if exists) that the new product performs signi􀀌cantly better than the existing one, as early as possible. Consider a ride-hailing company (e.g., Uber) . Suppose some passengers are in the recession state (at a high risk of stopping using the company's app) and the company comes up with certain strategy to intervene the recession process. We would like to test if there are some subgroups that are sensitive to the strategy and pin-point these subgroups if exists. It  \n􀀍c2021 Chengchun Shi, Shikai Luo, Hongtu Zhu and Rui Song.  \nLicense: CC-BY 4.0, see [https://creativecommons.org/licenses/by/4.0/](https://creativecommons.org/licenses/by/4.0/. Attribution)[. Attribution](https://creativecommons.org/licenses/by/4.0/. Attribution) requirements are provided  \nat [http://jmlr.org/papers/v22/21-0383.html](http://jmlr.org/papers/v22/21-0383.html).  \nShi, Luo, Zhu and Song  \nmotivates us to consider the null hypothesis that the treatment e􀀋ect is nonpositive for all passenger.  \nSuch a null hypothesis is closely related to the notion of qualitative treatment e􀀋ects in medical studies (QTE, Gail and Simon, 1985; Gunter et al. , 2007, 2011; Roth and Simon, 2018; Shi et al. , 2020b), and conditional moment inequalities in economics (see for example, Andrews and Shi, 2013, 2014; Chernozhukov et al., 2013; Armstrong and Chan, 2016; Changet al., 2015; Hsu, 2017) . However, these tests are computed o􀀏ine and might not be suitable to implement in online settings. Moreover, it is assumed in those papers that observations are independent. In online experiment, one may wish to adaptively allocate the treatment based on the observed data stream in order to maximize the cumulative reward or to detect the alternative more e","cbCaiaUiOVzTansm","https://ap.wps.com/l/cbCaiaUiOVzTansm","pdf",1082822,51,"English","# Introduction\n## Background and motivation\n## Related work and differences\n## Contributions","[{\"question\":\"What problem does the paper address in online A/B testing?\",\"answer\":\"It addresses how to detect qualitative treatment effects, where the new product performs significantly better only under certain conditions, not just on average.\"},{\"question\":\"How does the proposed method control error in sequential online experiments?\",\"answer\":\"It uses an online updating algorithm with adaptive randomization, sequential monitoring, and guaranteed type-I error control based on a spending-function style approach.\"},{\"question\":\"Why can’t earlier heterogeneous treatment effect tests be directly applied here?\",\"answer\":\"Heterogeneous treatment effects (HTE) and qualitative treatment effects (QTE) are related but fundamentally different hypotheses; cases exist where HTE holds while QTE does not.\"}]","An Online Sequential Test for Qualitative Treatment Effects - Research Paper | PDF",1788542875,129]