[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-581733-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-581733-en":130},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","generating-social-and-emotional-skill-items-humans-vs-chatgpt-reliability-and-validity-comparison-study","Generating Social and Emotional Skill Items: Humans vs. ChatGPT - Reliability and Validity Comparison Study","","Study examines whether ChatGPT can generate social and emotional (SE) skill assessment items and how a ChatGPT-generated assessment compares with a human-generated one. Large language models are positioned as tools for tasks such as summarizing, translating, rephrasing, and answering, while prior work has explored SE measures through personas and cheating contexts. The current work creates Likert and forced-choice items targeting conscientiousness, agreeableness, and emotional stability, then evaluates reliability and validity.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/generating-social-and-emotional-skill-items-humans-vs-chatgpt-reliability-and-validity-comparison-study/581733/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/generating-social-and-emotional-skill-items-humans-vs-chatgpt-reliability-and-validity-comparison-study/581733.png","ImageObject",300,407,{"name":42,"@type":43},"Aurelia","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-15","2026-10-15",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",7,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What is the study trying to achieve with ChatGPT in SE assessment development?","Question",{"text":63,"@type":64},"The study creates SE skill measures with ChatGPT and then compares a ChatGPT-generated assessment with a human-generated assessment in terms of reliability and validity.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How were Likert and forced-choice items generated in the item creation phase?",{"text":68,"@type":64},"ChatGPT and two human item writers followed the same instructions to generate Likert and forced-choice items covering conscientiousness, agreeableness, and emotional stability.",{"name":70,"@type":61,"acceptedAnswer":71},"What preliminary differences were observed between human-written and ChatGPT-written items?",{"text":72,"@type":64},"Human writers produced some reverse-keyed items, while ChatGPT did not. ChatGPT also produced double-barreled and multidimensional items, and its items tended to be longer with higher reading difficulty.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},581733,1792191036,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,110,114,119,122,126],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":55,"doc_module":4,"doc_module_name":25,"category_name":107,"show_sort_weight":108,"slug":109},"Healthcare",40,"healthcare",{"id":111,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":112,"slug":113},8,30,"research-report",{"id":115,"doc_module":4,"doc_module_name":25,"category_name":116,"show_sort_weight":117,"slug":118},9,"Religion & Spirituality",20,"religion-spirituality",{"id":117,"doc_module":4,"doc_module_name":25,"category_name":120,"show_sort_weight":117,"slug":121},"World Cup","world-cup",{"id":123,"doc_module":4,"doc_module_name":25,"category_name":124,"show_sort_weight":123,"slug":125},10,"Lifestyle","lifestyle",{"id":127,"doc_module":4,"doc_module_name":25,"category_name":128,"show_sort_weight":97,"slug":129},19,"General","general",{"code":4,"msg":82,"data":131},{"doc_id":79,"user_id":132,"nickname":42,"user_avatar":133,"doc_module":4,"category_id":111,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":111,"language":139,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":12,"update_tm":143,"read_time":117,"preview_page_count":4},1099514068365,"https://ap-avatar.wpscdn.com/avatar/10000253d8d9f28188e?x-image-process=image/resize,m_fixed,w_180,h_180&k=1794309523707217728","Generating Social and Emotional Skill Items: Humans vs . ChatGPT  \nKate E. Walton and Cristina Anguiano-Carrasco  \nLarge language models (LLMs) , such as ChatGPT, are becoming increasingly prominent. Their use is becoming more and more popular to assist with simple tasks , such as summarizing documents, translating languages , rephrasing sentences , or answering questions. Reports like McKinsey’s (Chui , & Yee, 2023) estimate that by implementing LLMs , corporations could see a potential growth of $4.4 trillion annually in corporate benefits, while Nielsen (2023) estimates a 66% increase in employee productivity when using LLMs and other forms of generative artificial intelligence (AI) . Can we use ChatGPT in the field of social and emotional learning assessment development to enhance our productivity?  \nSome have examined how social and emotional (SE) skills are related to ChatGPT usage, such as cheating in the academic domain (Greitemeyer & Kastenmüller, 2023) . In another study, researchers (de Winter et al. , 2023) had ChatGPT generate a large number of personas and complete several SE skill measures. They then carried out several analyses such as a factor analysis and correlations with outcome measures and determined how similar the results were to previous research using human-completed SE skill measures. In the current study, rather than have ChatGPT complete SE skill measures, we sought to have ChatGPT create SE skill measures. Ultimately, we will compare a ChatGPT-generated assessment with a humangenerated assessment in terms of reliability and validity.  \nThe Current Study  \nPhase 1: Item Generation  \nWe gave ChatGPT and two human item writers the same instructions to write Likert and forced choice (FC) items to cover the domains of conscientiousness (i.e. , the tendency to be persistent, reliable, dependable, etc. ), agreeableness (i.e. , the tendency to be empathic, helpful, trustworthy, etc. ) , and emotional stability (i.e. , the tendency to be stress tolerant, calm, poised, etc. ) . The humans were PhD-level subject matter experts (SMEs) . An example of a Likert item is a statement such as, I check my work before turning it in (an indicator of conscientiousness) . Respondents indicate on a scale of 1–6 how much they agree or disagree with the statement. An example of an FC triad is three statements such as, I check my work before turning it in, Iam a good team player (an indicator of agreeableness), and I can handle stress well (an indicator of emotional stability) . Respondents select the item that is most like them and least like them.  \nSimply, the instructions for generating Likert items were: Create six Likert conscientiousness items. The human writers and ChatGPT were then asked to create six agreeableness items and six emotional stability items. The instructions for generating FC items were: Create five  \n© 2024 by ACT, Inc. All rights reserved. | R2404  \nmultidimensional forced choice triads to measure conscientiousness, agreeableness, and emotional stability. Sample items created can be found in Table 1.  \nTable 1. Sample Items Written by Humans and ChatGPT  \nHumanGenerated Likert Items  \nI always have my school materials organized so I can easily find them.  \nIt’s hard for me to get along with some people. It’s easy for me to move on from sadness.  \nConscientiousness  \nAgreeableness Emotional Stability  \nChatGPTGenerated Likert Items  \nI find it challenging to relax until I’ve completed all of my tasks for the day.  \nI believe that cooperation and teamwork lead to better outcomes than individual efforts.  \nI can stay composed and rational even in highly stressful situations.  \nConscientiousness  \nAgreeableness Emotional Stability  \nHumanGenerated Forced Choice Items  \nI follow through on my commitments. I like to give others compliments.  \nI often have intense emotions.  \nConscientiousness Agreeableness  \nEmotional Stability  \nChatGPTGenerated Forced Choice Items  \nI am known for being responsible and ","cbCaicTm7KlwKuOP","https://ap.wps.com/l/cbCaicTm7KlwKuOP","pdf",316391,"English","# The purpose of the study\n# The current study\n## Phase 1: Item generation\n## Phase 2: Preliminary observations and analyses\n## Phase 3: Subject matter expert rating","[{\"question\":\"What is the study trying to achieve with ChatGPT in SE assessment development?\",\"answer\":\"The study creates SE skill measures with ChatGPT and then compares a ChatGPT-generated assessment with a human-generated assessment in terms of reliability and validity.\"},{\"question\":\"How were Likert and forced-choice items generated in the item creation phase?\",\"answer\":\"ChatGPT and two human item writers followed the same instructions to generate Likert and forced-choice items covering conscientiousness, agreeableness, and emotional stability.\"},{\"question\":\"What preliminary differences were observed between human-written and ChatGPT-written items?\",\"answer\":\"Human writers produced some reverse-keyed items, while ChatGPT did not. ChatGPT also produced double-barreled and multidimensional items, and its items tended to be longer with higher reading difficulty.\"}]","Generating Social and Emotional Skill Items: Humans vs. ChatGPT - Reliability and Validity Comparison Study | PDF",1792041290]