[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-153170-105":59,"doc-detail-153170-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","item-response-theory-for-nlp-eacl2024-tutorial","Item Response Theory for NLP EACL2024 Tutorial","","This document introduces Item Response Theory (IRT) for Natural Language Processing (NLP) at the EACL2024 tutorial held on March 21, 2024. It is presented by John P. Lalor, Pedro Rodriguez, João Sedoc, and Jose Hernandez-Orallo. The tutorial covers the motivation behind using IRT in NLP, introduces fundamental IRT models, discusses their application with artificial crowds, and highlights the py-irt package. The session also delves into the differences observed in examples across various NLP tasks, including Natural Language Inference (NLI) and Sentiment Analysis (SA). IRT is explained as a psychometric approach to measure latent traits of both test-takers and test questions, building instruments for measurement and developing theoretical approaches to measurement. The content is structured to provide a comprehensive overview of IRT's relevance and application in modern NLP research and development, aiming to equip attendees with the knowledge to leverage IRT for analyzing and improving NLP models and datasets.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/item-response-theory-for-nlp-eacl2024-tutorial/153170/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/item-response-theory-for-nlp-eacl2024-tutorial/153170.png","ImageObject",300,407,{"name":92,"@type":93},"Mali","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-09-18","2026-08-27",true,{"@type":102,"interactionType":103,"userInteractionCount":34},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What is Item Response Theory (IRT)?","Question",{"text":112,"@type":113},"IRT is a psychometric approach used to measure the latent traits of test-takers and test questions, also known as 'items'.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What NLP tasks are discussed in relation to IRT?",{"text":117,"@type":113},"The tutorial discusses IRT's application in tasks such as Natural Language Inference (NLI) and Sentiment Analysis (SA), highlighting differences in examples and difficulty levels.",{"name":119,"@type":110,"acceptedAnswer":120},"What is the purpose of the py-irt package mentioned?",{"text":121,"@type":113},"The py-irt package is a tool highlighted in the tutorial that facilitates the application and study of IRT models within the context of NLP.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},153170,1787871734,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":34,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":129,"read_time":144},2336475104362,"https://ap-avatar.wpscdn.com/avatar/22000c4c46a41b752dd?x-image-process=image/resize,m_fixed,w_180,h_180&k=1786595829695023868","Item Response Theory for NLP  \nEACL2024 Tutorial, 21 st March 2024  \n,  \nJohn P. Lalor, Pedro Rodriguez , João S edo c, Jose Hernandez-Oral lo  \n[htt ps : / / ea c l 202 4i r t . g i t h ub. i o/](htt ps : / / ea c l 202 4i r t . g i t h ub. i o/)  \nIn this session  \nMotivation  \nIntroducing IRT  \nIRT Models with Artificial Crowds  \nThe py-irt Package  \nMotivation  \nDifferences between Examples  \nNatural language inference (NLI)  \n\n| Premise Hypothesis Label Difficulty |  |  |  |  |\n| --- | --- | --- | --- | --- |\n| A little girl eating a sucker\u003Cbr>People were watching the tournament in the stadium\u003Cbr>Two girls on a bridge dancing with the city skyline in the background | A child eating candy\u003Cbr>The people are sitting outside on the grass\u003Cbr>The girls are sisters. | Entailment Contradiction\u003Cbr>Neutral |  | easy\u003Cbr>hard\u003Cbr>easy |\n| Sentiment analysis (SA) |  |  |  |  |\n| Phrase |  |  | Label | Difficulty |\n| The stupidest, most insulting movie of 2002’s first quarter.\u003Cbr>Still, it gets the job done-a sleepy afternoon rental.\u003Cbr>An endlessly fascinating, landmark movie that is as bold as anything the cinema has seen in years. Perhaps no picture ever made has more literally showed that the road to hell is paved with good intentions. |  |  | Negative\u003Cbr>Negative\u003Cbr>Positive\u003Cbr>Positive | easy\u003Cbr>hard\u003Cbr>easy\u003Cbr>hard |\n\nLeaderboards  \n[https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)  \nDifferences in Questions  \nDifferences in Questions  \nDifferences in Questions  \nSource: Boyd-Graber and Börschinger (2020)  \nIntroducing IRT  \nPsychometrics  \nPsychometrics: study of quantitative measurement practices  \n• Building instruments for measurement (standardized tests)  \n• Development of theoretical approaches to measurement  \nItem Response Theory (IRT): measure latent traits of test-takers and test questions (“items”)","cbCaie8UzIZaPvSN","https://ap.wps.com/l/cbCaie8UzIZaPvSN","pdf",1191318,35,"English","# Session Outline\n## Motivation\n## Introducing IRT\n## IRT Models with Artificial Crowds\n## The py-irt Package\n## Motivation\n## Differences between Examples\n## Natural language inference (NLI)\n## Sentiment analysis (SA)","[{\"question\":\"What is Item Response Theory (IRT)?\",\"answer\":\"IRT is a psychometric approach used to measure the latent traits of test-takers and test questions, also known as 'items'.\"},{\"question\":\"What NLP tasks are discussed in relation to IRT?\",\"answer\":\"The tutorial discusses IRT's application in tasks such as Natural Language Inference (NLI) and Sentiment Analysis (SA), highlighting differences in examples and difficulty levels.\"},{\"question\":\"What is the purpose of the py-irt package mentioned?\",\"answer\":\"The py-irt package is a tool highlighted in the tutorial that facilitates the application and study of IRT models within the context of NLP.\"}]","Item Response Theory for NLP EACL2024 Tutorial | PDF",88]