[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-515938-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-515938-en":130},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","societal-impacts-research-requires-benchmarks-for-creative-composition-tasks","Societal Impacts Research Requires Benchmarks for Creative Composition Tasks","","Foundation models that automate cognitive tasks create major technological change, but their broader societal consequences are not yet well understood. The work warns that these systems may contribute formulaic, homogeneous, and potentially misleading synthetic content. It develops benchmark motivation by analyzing 2 million language-model user prompts and shows creative composition tasks are a prevalent category where users need everyday creativity for personal goals.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/societal-impacts-research-requires-benchmarks-for-creative-composition-tasks/515938/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/societal-impacts-research-requires-benchmarks-for-creative-composition-tasks/515938.png","ImageObject",300,407,{"name":42,"@type":43},"Aria Callaghan","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-15","2026-10-06",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",5,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"Why do current benchmarks not fully ensure desired behavior at deployment?","Question",{"text":63,"@type":64},"Even if a model meets a threshold on specific safety or harm-related benchmarks, it does not guarantee the behavior will fully appear in real deployment settings. A key barrier is the gap between how models are used and how benchmarks are developed.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"What usage category does the paper identify as most prevalent for creative help?",{"text":68,"@type":64},"Through thematic analysis of 2 million prompts, the paper identifies creative composition tasks as a common usage category where users seek support for personal tasks requiring everyday creativity.",{"name":70,"@type":61,"acceptedAnswer":71},"How does the paper connect creative composition tasks to societal harms?",{"text":72,"@type":64},"The paper argues the same use cases that lack thorough evaluations can still produce negative downstream impacts. It motivates benchmarks focused on creative composition tasks to better measure both progress and harms of models with creative capabilities.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},515938,1792188835,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,100,105,110,114,119,122,126],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":55,"doc_module":4,"doc_module_name":25,"category_name":97,"show_sort_weight":98,"slug":99},"Comic",60,"comic",{"id":101,"doc_module":4,"doc_module_name":25,"category_name":102,"show_sort_weight":103,"slug":104},6,"Technology",50,"technology",{"id":106,"doc_module":4,"doc_module_name":25,"category_name":107,"show_sort_weight":108,"slug":109},7,"Healthcare",40,"healthcare",{"id":111,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":112,"slug":113},8,30,"research-report",{"id":115,"doc_module":4,"doc_module_name":25,"category_name":116,"show_sort_weight":117,"slug":118},9,"Religion & Spirituality",20,"religion-spirituality",{"id":117,"doc_module":4,"doc_module_name":25,"category_name":120,"show_sort_weight":117,"slug":121},"World Cup","world-cup",{"id":123,"doc_module":4,"doc_module_name":25,"category_name":124,"show_sort_weight":123,"slug":125},10,"Lifestyle","lifestyle",{"id":127,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":55,"slug":129},19,"General","general",{"code":4,"msg":82,"data":131},{"doc_id":79,"user_id":132,"nickname":42,"user_avatar":133,"doc_module":4,"category_id":111,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":139,"language":140,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":12,"update_tm":144,"read_time":145,"preview_page_count":33,"preview_urls":146},962084926284,"https://ap-avatar.wpscdn.com/davatar_29158cc5080c5b710cf443261637dec0","Position: Societal Impacts Research Requires Benchmarks for Creative Composition Tasks  \nJudy Hanwen Shen 1 Carlos Guestrin 1  \narXiv :2504 .06549v2 [ cs .CY] 24 May 2025  \nAbstract  \nFoundation models that are capable of automating cognitive tasks represent a pivotal technological shift, yet their societal implications remain unclear. These systems promise exciting advances, yet they also risk flooding our information ecosystem with formulaic, homogeneous, and potentially misleading synthetic content. Developing benchmarks grounded in real use cases where these risks are most significant is therefore critical. Through a thematic analysis using 2 million language model user prompts, we identify creative composition tasks as a prevalent usage category where users seek help with personal tasks that require everyday creativity. Our fine-grained analysis identifies mismatches between current benchmarks and usage patterns among these tasks.  \nCrucially, we argue that the same use cases that currently lack thorough evaluations can lead to negative downstream impacts. This position paper argues that benchmarks focused on creative composition tasks is a necessary step towards understanding the societal harms of AI-generated content. We call for greater transparency in usage patterns to inform the development of new benchmarks that can effectively measure both the progress and the impacts of models with creative capabilities.  \n1. Introduction  \nResponsible and safe AI development has been significantly aided by the availability and adoption of key benchmarks (Buolamwini & Gebru, 2018 ; Angwin et al., 2022 ; Weidinger et al., 2023) . For example, the UCI Adult (Becker & Kohavi, 1996) (and later Folktables (Ding et al., 2021)) dataset has served as a workhorse of measuring demographic  \n1Department of Computer Science, Stanford University. Correspondence to: Judy Hanwen Shen \u003C[jhshen@stanford.edu](jhshen@stanford.edu) >.  \nProceedings of the 42 nd International Conference on Machine Learning, Vancouver, Canada. PMLR 267, 2025 . Copyright 2025 by the author(s) .  \ndisparities in predicting income both for early fairnessaware algorithms and in large language model trustworthiness benchmarks today (Huang et al., 2024 ; Wang et al., 2023) . As another example, TruthfulQA, a benchmark to measure truthfulness (lack of misconceptions) in English, has been included in benchmark suites (Huang et al., 2024), leaderboards (Liang et al., 2022), and model technical reports (Achiam et al., 2023 ; Touvron et al., 2023) .  \nThese are examples of benchmarks used to measure a specific construct related to safety or harm mitigation (Rttger et al., 2024) . However, when a model meets a certain score threshold on these benchmarks, there is no guarantee that at deployment time the model will exhibit the desired behavior entirely (Kaiyom et al., 2024) . One barrier to generalizable evaluation is the gap between model usage and benchmark development. Recent works have highlighted the need for deployment-based evaluations for AI systems (De Vries et al., 2020 ; Berente et al., 2024 ; Saxon et al., 2024), aligning with calls for benchmarks that demonstrate stronger construct validity through contextual considerations (Rajiet al., 2021) . In the domain of societal impacts, Wang et al.(2024) extends this reasoning to advocate for more comprehensive evaluation benchmark suites.  \nWhat are the deployment scenarios where assessing the societal impacts of AI is important? One way to answer this question is to study how models are used. Domainspecific studies have found an increase in the use of large language models in writing; from student essays (Jelson & Lee, 2024) to scientific articles (Liang et al., 2024) . Meanwhile, language model prompt datasets report categories such as “helping and creative writing”(Zhao et al., 2024) and “language and content creation”(Zheng et al., 2023) contributing to up to two-thirds of usage. We translate these broad domains into fine","cbCaikgl5bMCBGl1","https://ap.wps.com/l/cbCaikgl5bMCBGl1","pdf",374075,21,"English","# Introduction\n## Benchmarks and deployment misalignment\n## Societal impacts via real usage patterns\n## Creative composition tasks as a key category","[{\"question\":\"Why do current benchmarks not fully ensure desired behavior at deployment?\",\"answer\":\"Even if a model meets a threshold on specific safety or harm-related benchmarks, it does not guarantee the behavior will fully appear in real deployment settings. A key barrier is the gap between how models are used and how benchmarks are developed.\"},{\"question\":\"What usage category does the paper identify as most prevalent for creative help?\",\"answer\":\"Through thematic analysis of 2 million prompts, the paper identifies creative composition tasks as a common usage category where users seek support for personal tasks requiring everyday creativity.\"},{\"question\":\"How does the paper connect creative composition tasks to societal harms?\",\"answer\":\"The paper argues the same use cases that lack thorough evaluations can still produce negative downstream impacts. It motivates benchmarks focused on creative composition tasks to better measure both progress and harms of models with creative capabilities.\"}]","Societal Impacts Research Requires Benchmarks for Creative Composition Tasks | PDF",1791323748,53,[147,151,155,159],{"page_no":22,"url":148,"width":149,"height":150},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/6df4840cc5c484adf4837606f8b34497.webp",983,1272,{"page_no":26,"url":152,"width":153,"height":154},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/29664ac713ded6e05f3cbc806e394b92.webp",982,1270,{"page_no":30,"url":156,"width":157,"height":158},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/4a9f638d708dd7a57376f51c972af093.webp",981,1269,{"page_no":33,"url":160,"width":161,"height":162},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/41e71011a190be037fe1d6d896a6024b.webp",980,1268]