[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-327243-105":59,"doc-detail-327243-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","adversarial-pragmatics-for-ai-safety-evaluation-a-benchmark-for-instruction-conflict-embedded-commands-and-policy-ambiguity","Adversarial Pragmatics for AI Safety Evaluation - A Benchmark for Instruction Conflict, Embedded Commands, and Policy Ambiguity","","Safety evaluations for language models increasingly hinge on ambiguous natural-language behavior: whether a model followed an instruction, refused appropriately, complied with a policy, resisted an embedded command, or misreported progress in agentic work. Existing benchmarks often collapse these factors into pass/fail labels, hiding whether failures stem from capability limits, policy ambiguity, instruction conflict, scaffold breakdown, or evaluator instability. The paper proposes adversarial pragmatics as a benchmark and annotation protocol, with a controlled taxonomy, seed and pilot evaluations, expert protocol, and metrics for judge validity and taxonomy drift.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/adversarial-pragmatics-for-ai-safety-evaluation-a-benchmark-for-instruction-conflict-embedded-commands-and-policy-ambiguity/327243/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/adversarial-pragmatics-for-ai-safety-evaluation-a-benchmark-for-instruction-conflict-embedded-commands-and-policy-ambiguity/327243.png","ImageObject",300,407,{"name":92,"@type":93},"Lucas Martin","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-07","2026-09-21",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",21,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"Why do pass/fail labels lose information in AI safety evaluation?","Question",{"text":113,"@type":114},"They are too coarse when the same failure outcome can reflect different underlying causes, such as capability limits, instruction conflict, policy ambiguity, tool or scaffold errors, goal drift, or misreporting.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"What does the paper mean by “adversarial pragmatics”?",{"text":118,"@type":114},"It targets safety-relevant model behavior in cases where instruction status, source authority, quotation, scope, reference, speech-act force, or policy category must be inferred from language use to diagnose failure.",{"name":120,"@type":111,"acceptedAnswer":121},"What does the framework include for the benchmark and evaluation?",{"text":122,"@type":114},"It provides a linguistically controlled taxonomy, an 18-item seed benchmark with validator-enforced metadata, a local seed pilot, an expert-evaluation protocol with multiple judgment dimensions, and metrics for judge validity and diagnostic ambiguity, plus tools and reproducibility commands.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},327243,1790487716,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":145,"read_time":26,"preview_page_count":4},8796095360427,"https://ap-avatar.wpscdn.com/davatar_994ba38a5ba835b3df7d355c54d3ed8d","arXiv :2607 .0 1 153v 1 [ cs .CL] 1 Jul 2026  \nAdversarial Pragmatics for AI Safety Evaluation: A Benchmark for Instruction Conflict, Embedded Commands, and Policy Ambiguity  \nBrett Reynolds *  \nHumber Polytechnic & University of Toronto July 2026  \nAbstract  \nSafety evaluations for language models increasingly depend on judgments about ambiguous natural-language behaviour: whether a model has followed an instruction, refused appropriately, complied with a policy, resisted an embedded command, or misreported progress in anagentic task. Existing benchmarks often compress these distinctions into pass/fail labels, obscuring whether failures arise from capability limits, policy ambiguity, instruction conflict, scaffold failure, or unstable evaluator judgments.  \nThis paper introduces adversarial pragmatics as a benchmark and annotation protocol for evaluating model behaviour under instruction conflict, embedded commands, quotation, scope ambiguity, deixis, indirect speech acts, and multi-turn agent transcripts. The contribution is empirical and methodological: a linguistically controlled taxonomy, an 18-item seed benchmark with validator-enforced metadata, a 54-row local seed pilot, an expert-evaluation protocol distinguishing task success, policy compliance, safety risk, refusal outcome, and evaluator confidence, and metrics for judge validity, diagnostic ambiguity, and taxonomy drift. The framework turns linguistic judgment methodology into a practical tool for validating safety evals, LLM judges, gold-set construction, prompt-injection tests, and safety documentation.  \nKeywords: AI safety evaluation; adversarial pragmatics; instruction hierarchy; prompt injection; annotation protocols; benchmark validity; LLM judges  \nSupplementary material: A separate supplement documents the seed schema, item inventory, rater protocol, reproducibility commands, and sanitized local-pilot summaries.  \n1 The Evaluation Problem  \nFrontier-model safety evaluation increasingly evaluates language-mediated control. The central question isn’t only whether a model can solve a task. It also asks whether the model can identify which  \n* Contact: [brett.reynolds@humber.ca](brett.reynolds@humber.ca)  \n2 WHY PASS/FAIL LABELS LOSE INFORMATION 2  \nnatural-language material has directive force, which policy boundary applies, and what a transcript shows about the model’s behaviour.  \nThe problem already appears in work on holistic language-model evaluation, which treats model assessment as a multi-scenario, multi-metric problem rather than a single accuracy number (Lianget al., 2022) . Safety evaluation adds a sharper version of the same problem: the relevant behaviour is often not just an answer, but the model’s uptake ofan instruction, policy, or source relation.  \nThe practical bottleneck is visible in prompt injection, model-policy evaluation, red-team triage, LLM-as-judge grading, and agent transcript review. Prompt-injection research shows that LLMintegrated applications can blur the boundary between data and instructions (Greshake et al., 2023); instruction-hierarchy work frames the same vulnerability as failure to follow privileged rather than lower-priority instructions (Wallace et al., 2024) . Agent evaluations make the issue harder because untrusted content can arrive through tools and retrieved documents rather than through the user turn itself (Debenedetti et al., 2024) .  \nIn each setting, a surface string may be an instruction, a quotation, a cited passage, a tool output, a policy example, a user request, or an adversarial attempt to change authority. A pass/fail label loses information when it doesn’t say which role the string had and why.  \nEvaluation labels function here as inference licenses. A label attached to an item states what nearby judgmentsit supports: how the case should behave under paraphrase, wrapper change, model change, or judge-prompt change. Projectibility (what observing some features licenses us to predict about others; (Go","cbCaiknrls42UDJh","https://ap.wps.com/l/cbCaiknrls42UDJh","pdf",353197,24,"English","# Abstract\n# The Evaluation Problem\n## Why Pass/Fail Labels Lose Information\n## Artifact and pilot findings","[{\"question\":\"Why do pass/fail labels lose information in AI safety evaluation?\",\"answer\":\"They are too coarse when the same failure outcome can reflect different underlying causes, such as capability limits, instruction conflict, policy ambiguity, tool or scaffold errors, goal drift, or misreporting.\"},{\"question\":\"What does the paper mean by “adversarial pragmatics”?\",\"answer\":\"It targets safety-relevant model behavior in cases where instruction status, source authority, quotation, scope, reference, speech-act force, or policy category must be inferred from language use to diagnose failure.\"},{\"question\":\"What does the framework include for the benchmark and evaluation?\",\"answer\":\"It provides a linguistically controlled taxonomy, an 18-item seed benchmark with validator-enforced metadata, a local seed pilot, an expert-evaluation protocol with multiple judgment dimensions, and metrics for judge validity and diagnostic ambiguity, plus tools and reproducibility commands.\"}]","Adversarial Pragmatics for AI Safety Evaluation - A Benchmark for Instruction Conflict, Embedded Commands, and Policy Ambiguity | PDF",1789976792]