[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-657539-105":59,"doc-detail-657539-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","llms-as-assistants-for-data-annotation-addressing-disagreement-and-supporting-expert-processes","LLMs as Assistants for Data Annotation - Addressing Disagreement and Supporting Expert Processes","","This paper investigates how Large Language Models can support human data annotation pipelines, especially by enabling expert-informed annotation guideline development for document-level content categorisation. Three experiments evaluate LLM roles in annotation: as annotators, as domain experts to assist disagreement resolution, and as analysts of annotator discussions. Using GPT-4.5 and Claude Sonnet 4, results indicate mixed-to-positive usefulness, particularly when rich context such as prior annotations and discussion history is provided.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/llms-as-assistants-for-data-annotation-addressing-disagreement-and-supporting-expert-processes/657539/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/llms-as-assistants-for-data-annotation-addressing-disagreement-and-supporting-expert-processes/657539.png","ImageObject",300,407,{"name":92,"@type":93},"\tCallum ","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-29","2026-10-26",true,{"@type":102,"interactionType":103,"userInteractionCount":29},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What are the three experiments that use LLMs in the annotation pipeline?","Question",{"text":112,"@type":113},"The paper explores LLMs as annotators, as domain experts supporting disagreement resolution, and as analysts of annotator discussions.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"Which metrics or criteria are used to evaluate the LLM-generated guidelines?",{"text":117,"@type":113},"The evaluation focuses on coverage, applicability, and usefulness for document-level classification tasks.",{"name":119,"@type":110,"acceptedAnswer":120},"What contextual information improves the effectiveness of LLM support?",{"text":121,"@type":113},"The paper reports better support when LLMs are given rich context such as prior human annotations and annotator discussions.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},657539,1792982553,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":29,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":129,"read_time":144},137451211410,"https://ap-avatar.wpscdn.com/avatar/2000bb0a9246f588df?x-image-process=image/resize,m_fixed,w_180,h_180&k=1786362646172706240","LLMs as Assistants for Data Annotation: Addressing Disagreement  \nand Supporting Expert Processes  \nMark Andrade, Bláithín Heffernan, Abigail Walsh, Sheila Castilho  \nADAPT Centre, Dublin City University  \n[mark.andrade@adaptcentre.ie](mark.andrade@adaptcentre.ie), [heffernanblaithin@gmail.com](heffernanblaithin@gmail.com),  \n[abigail.walsh@adaptcentre.ie](abigail.walsh@adaptcentre.ie), [sheila.castilho@dcu.ie](sheila.castilho@dcu.ie)  \nAbstract  \nThis paper investigates the potential of Large Language Models to assist human annotation pipelines, with a particular focus on supporting the development of expert-informed annotation guidelines for document-level content categorisation. We present three experiments exploring distinct roles for LLMs in annotation: as annotators, as domain experts assisting in disagreement resolution, and as analysts of annotator discussions. Using GPT-4 .5 and Claude Sonnet 4, we evaluate LLM-generated annotation guidelines for a document-level classification tasks in terms of coverage, applicability, and usefulness. Preliminary results are mixed-to-positive, with evidence that LLMs can provide useful support across different stages of the annotation pipeline, particularly when supplied with rich contextual information such as prior human annotations and annotator discussions.  \nKeywords: LLM-assisted annotation, annotation guidelines, annotation pipeline  \n1. Introduction  \nData annotation is a central component of Natural Language Processing (NLP), providing the labelled data necessary for training, evaluating, and analysing computational models (Fort , 2016) . Across tasks such as classification, information extraction, and discourse analysis, annotation enables the operationalisation of linguistic and semantic phenomena into structured representations that can be processed computationally. Annotation remains a resource-intensive and complex process, often requiring expert knowledge, continuous discussion among annotators to resolve disagreements and align interpretations, careful guideline development, and several rounds of validation to ensure consistency and reliability.  \nIn this context, Large Language Models (LLMs) are rapidly reshaping the landscape of the NLP field, by offering scalable, cost-effective, and accessible solutions. Their adoption across research domains has accelerated in recent years, particularly for text classification, evaluation, sentiment analysis, and other annotation-heavy tasks. On the one hand, LLMs can outperform or complement human annotators in speed and accuracy (Gilardiet al. , 2023 ; Goel et al. , 2023), and fine-tuned opensource models have shown promising performance across diverse applications. On the other hand, their outputs are highly sensitive to implementation choices such as model selection, prompting strategy, and parameter settings, leading to risks of bias, reproducibility issues, and “LLM hacking”(Pavlovic and Poesio, 2024 ; Baumann et al. , 2025) . Furthermore, the absence of established standards and best practices has raised concerns about the overall quality and validity of LLM-based annotation  \nresearch (Törnberg, 2024) . Annotation outputs are also prone to format inconsistencies, invalid labels, or deviations from instructions, which can undermine downstream analysis if not properly managed.  \nGiven these opportunities and challenges, thereis a growing need to critically examine the role of LLMs in data annotation. This work is motivated by the need to empirically test a content profiling framework proposed by Castilho and O’Brien (2026), which encompasses multiple dimensions, including content, topic, genre, text type, style, register, and domain (section 3.1) . We explore the potential of LLMs in assisting the human annotation when annotating a subset of this framework (content, genre, text type, and topic) in English. We aim to investigate whether LLMs can further reduce cognitive and labour load, allowing for more productive annotatio","cbCaic2zvye6Im0U","https://ap.wps.com/l/cbCaic2zvye6Im0U","pdf",289114,11,"English","# Introduction\n# Related Work\n## Effective Annotation Pipeline","[{\"question\":\"What are the three experiments that use LLMs in the annotation pipeline?\",\"answer\":\"The paper explores LLMs as annotators, as domain experts supporting disagreement resolution, and as analysts of annotator discussions.\"},{\"question\":\"Which metrics or criteria are used to evaluate the LLM-generated guidelines?\",\"answer\":\"The evaluation focuses on coverage, applicability, and usefulness for document-level classification tasks.\"},{\"question\":\"What contextual information improves the effectiveness of LLM support?\",\"answer\":\"The paper reports better support when LLMs are given rich context such as prior human annotations and annotator discussions.\"}]","LLMs as Assistants for Data Annotation - Addressing Disagreement and Supporting Expert Processes | PDF",28]