[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-273374-105":59,"doc-detail-273374-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","compared-to-what-baselines-and-metrics-for-counterfactual-prompting","Compared to What? - Baselines and Metrics for Counterfactual Prompting","","Counterfactual prompting, which perturbs a single factor to measure output change, is widely used to assess LLM bias and chain-of-thought faithfulness. This work shows that observed effects cannot be reliably attributed to the targeted factor without a baseline comparison using meaning-preserving edits that capture general model sensitivity. In causal inference terms, counterfactual edits act as compound treatments, combining the variable of interest with surface-form variation and violating treatment variation irrelevance.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/compared-to-what-baselines-and-metrics-for-counterfactual-prompting/273374/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/compared-to-what-baselines-and-metrics-for-counterfactual-prompting/273374.png","ImageObject",300,407,{"name":92,"@type":93},"Himbo","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-29","2026-09-15",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",16,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"Why can’t observed changes in counterfactual prompting be attributed solely to the targeted factor?","Question",{"text":113,"@type":114},"Because each counterfactual edit bundles the intended variable with incidental surface-form variation, so treatment variation irrelevance is violated. This means effects may reflect general sensitivity to input changes rather than the factor of interest.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"What baseline does the paper propose to make counterfactual results reliable?",{"text":118,"@type":114},"Compare differences under target interventions to a null distribution of differences induced by semantics-preserving paraphrases, adjusted to match perturbation token edit lengths, using statistical testing.",{"name":120,"@type":111,"acceptedAnswer":121},"What metric family is found to be most powerful in the proposed framework?",{"text":122,"@type":114},"Per-sample metrics such as JSD and KL are dramatically more powerful than aggregate metrics like MI, ϕ, and flip rate. Regression methods are also shown to uniquely and powerfully characterize effect direction and magnitude.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},273374,1789471136,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":130,"read_time":145,"preview_page_count":4},687197100911,"https://ap-avatar.wpscdn.com/avatar/a000239b6f1da00475?x-image-process=image/resize,m_fixed,w_180,h_180&k=1785132997149421697","arXiv : 2605 .0 1048v2 [cs .CL] 20 Aug 2026  \nCompared to What?  \nBaselines and Metrics for Counterfactual Prompting  \nZihao Yang1, Mosh Levy2, Yoav Goldberg2,3 & Byron C. Wallace1  \n1Northeastern University 2 Bar-Ilan University 3Allen Institute for AI [yang.zih@northeastern.edu](yang.zih@northeastern.edu)  \nAbstract  \nCounterfactual prompting (i.e., perturbing a single factor and measuring output change) is widely used to evaluate things like LLM bias and CoT faithfulness. But in this work we argue that observed effects cannot be reliably attributed to the targeted factor without accounting for baseline“meaning-preserving” modifications to text that establish general model sensitivity. In the parlance of causal inference, this is because every counterfactual edit is a compound treatment that bundles the variable of interest with incidental surface-form variation; this violates treatment variation irrelevance. For example, we observe prediction flip rates on MedQA of 14.9% when we surgically change patient gender. However, this is statistically indistinguishable from the flip rates induced by simply paraphrasing inputs (14.1%) . In this case, it would therefore be unwarranted to conclude that the LLM is especially sensitive to patient gender. To account for this and robustly measure the effects of targeted interventions, we propose a framework in which we compare (via statistical testing) differences observed under target interventions to those induced by paraphrasing inputs (adjusted to match perturbation token edit lengths) . We then use this framework to revisit a prior analysis done on the MedPerturb dataset, which reported evidence of model sensitivity to patient demographics and stylistic cues.  \nWe find that these effects largely dissipate when we account for general model sensitivity, with only 5 of 120 tests reaching statistical significance;  \nthis calls into question whether models are particularly sensitive to these factors. Applying the same framework to occupational biography classification (Bias-in-Bios), we detect clearly significant directional gender bias, showing that the framework identifies real directional effects even when they are small. We evaluate a range of metrics—aggregate, per-sample distributional, and regression—and find that per-sample metrics (JSD, KL) are dramatically more powerful than aggregate metrics (MI, ϕ, flip rate) and regression powerfully and uniquely characterizes effect direction and magnitude. We provide empirical guidance on running counterfactual prompting experiments for LLMs.1  \n1 Introduction  \nCounterfactual prompts are a practical means of evaluating LLM behavior. The idea is to surgically edit inputs to manipulate (only) a factor of interest, holding everything else constant: The intuition is that observed differences in outputs may be attributed to this factor. Counterfactuals have been used widely for the analysis of LLMs, from assessing demographic bias (Nangia et al., 2020; Rawat et al., 2024; Gourabathina et al., 2025) to testing whether chain-of-thought (CoT) reasoning is faithful (Turpin et al., 2023; Chen et al., 2025) .  \n1 Code to reproduce the experiments in this paper is available at [https://github.com/redagavin/](https://github.com/redagavin/)[ ](https://github.com/redagavin/)counterfactual-prompting-baselines. We released a Python package, cfprompt, implementing the framework: pip install cfprompt; [https://github.com/redagavin/cfprompt](https://github.com/redagavin/cfprompt).  \nA B  \nRepeat across 919 MedQA clinical vignettes (DeepSeek-R1 8B):  \nGender swap  \nBenign paraphrase  \n~~ ~~ 14.9%  \n~~ ~~ 14. 1%  \n0% 5% 10% 15% 20%  \nAnswer flip rate  \nFigure 1: An example illustrating the core problem. (A) A medical question is perturbed by both swapping the patient’s gender and applying a benign paraphrase, and the model flipsits answer in both cases. (B) Across 919 MedQA clinical vignettes, the two perturbation types produce comparable answer flip rates (14.9","cbCais2Ovqb9l6Re","https://ap.wps.com/l/cbCais2Ovqb9l6Re","pdf",508175,33,"English","# Abstract\n# Introduction\n## Counterfactual prompts as factor manipulation\n## Need for baseline semantics-preserving comparisons\n# Figure 1: Problem illustration\n## Gender swap vs benign paraphrase\n# MedQA experimental findings\n## Prediction flip-rate comparison","[{\"question\":\"Why can’t observed changes in counterfactual prompting be attributed solely to the targeted factor?\",\"answer\":\"Because each counterfactual edit bundles the intended variable with incidental surface-form variation, so treatment variation irrelevance is violated. This means effects may reflect general sensitivity to input changes rather than the factor of interest.\"},{\"question\":\"What baseline does the paper propose to make counterfactual results reliable?\",\"answer\":\"Compare differences under target interventions to a null distribution of differences induced by semantics-preserving paraphrases, adjusted to match perturbation token edit lengths, using statistical testing.\"},{\"question\":\"What metric family is found to be most powerful in the proposed framework?\",\"answer\":\"Per-sample metrics such as JSD and KL are dramatically more powerful than aggregate metrics like MI, ϕ, and flip rate. Regression methods are also shown to uniquely and powerfully characterize effect direction and magnitude.\"}]","Compared to What? - Baselines and Metrics for Counterfactual Prompting | PDF",83]