[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-203702-105":59,"doc-detail-203702-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","calibrating-factual-knowledge-in-pretrained-language-models-research-paper-december-2022","Calibrating Factual Knowledge in Pretrained Language Models - Research Paper - December 2022","","Pretrained Language Models can store factual knowledge, yet the facts inside them may be incorrect, limiting reliability in downstream applications. The work introduces CALINET, a lightweight, task-agnostic calibration approach that first detects wrong facts using a contrastive score between right and fake facts. If detection fails, CALINET adds and adapts new parameters for specific factual texts while keeping the original PLM fixed, improving knowledge probing and closed-book question answering.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/calibrating-factual-knowledge-in-pretrained-language-models-research-paper-december-2022/203702/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/calibrating-factual-knowledge-in-pretrained-language-models-research-paper-december-2022/203702.png","ImageObject",300,407,{"name":92,"@type":93},"Chloe Bennett","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-08","2026-09-04",true,{"@type":102,"interactionType":103,"userInteractionCount":52},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What problem does this paper address about pretrained language models?","Question",{"text":112,"@type":113},"Pretrained language models may store factual knowledge that is not always correct. The paper focuses on detecting and correcting false facts without retraining the whole model from scratch.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How does CALINET calibrate incorrect factual knowledge?",{"text":117,"@type":113},"CALINET uses a two-stage strategy: it first evaluates whether a PLM assigns higher scores to right facts than to plausible negative facts. Then it finetunes lightweight, newly added parameters for specific factual texts while keeping the original PLM parameters fixed.",{"name":119,"@type":110,"acceptedAnswer":120},"How is the calibration effectiveness evaluated?",{"text":121,"@type":113},"Experiments are conducted on knowledge probing tasks and closed-book question answering, showing that calibrated models are more accurate and can generalize knowledge after fine-tuning.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},203702,1788562709,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":52,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":129,"read_time":144},962084925782,"https://ap-avatar.wpscdn.com/davatar_9964176cb1d06d4a9deccf72a44ae3dc","Calibrating Factual Knowledge in Pretrained Language Models  \nQingxiu Dong 1 ∗ , Damai Dai 1 ∗ , Yifan Song 1 , Jingjing Xu2 , Zhifang Sui 1 and Lei Li3 1 MOE Key Lab of Computational Linguistics, School of Computer Science, Peking University  \n2 Shanghai AI Lab 3 University of California, Santa Barbara [dqx@stu.pku.edu.cn](dqx@stu.pku.edu.cn) , {daidamai,yfsong,jingjingxu, [szf}@pku.edu.cn](szf}@pku.edu.cn) ,  \n[lilei@cs.ucsb.edu](lilei@cs.ucsb.edu)  \nAbstract  \nPrevious literature has proved that Pretrained Language Models (PLMs) can store factual knowledge. However, we find that facts stored in the PLMs are not always correct. It motivates us to explore a fundamental question: How do we calibrate factual knowledge in PLMs without re-training from scratch? In this work, we propose a simple and lightweight method CALINET to achieve this goal. To be specific, we first detect whether PLMs can learn the right facts via a contrastive score between right and fake facts. If not, we then use a lightweight method to add and adapt new parameters to specific factual texts. Experiments on the knowledge probing task show the calibration effectiveness and efficiency. In addition, through closed-book question answering, we find that the calibrated PLM possesses knowledge generalization ability after fine-tuning. Beyond the calibration performance, we further investigate and visualize the knowledge calibration mechanism. The code and data are available at [https://github.com/dqxiu/CaliNet](https://github.com/dqxiu/CaliNet).  \n1 Introduction  \nRecently, Pretrained Language Models (PLMs) have improved performance on various Natural Language Processing (NLP) tasks (Devlin et al., 2019 ; Raffel et al., 2020 ; Brown et al., 2020) . Probing tasks like LAMA (Petroni et al., 2019 ; Elazar et al., 2021 ; Jiang et al., 2020) have shown that PLMs can store factual knowledge and act as knowledge bases. Leveraging knowledge in PLMscan benefit knowledge-intensive downstream tasks such as fact checking and question answering (Lee et al., 2020 ; Bouraoui et al., 2020 ; Roberts et al., 2020a) . However, knowledge stored in PLMs may have factual errors, which hinder the performance in downstream tasks (Elazar et al., 2021 ; Cao et al., 2021a) . It is essential and fundamental to detect and calibrate false facts stored in a PLM.  \n*Equal contribution.  \nFigure 1: Illustration of knowledge calibration. Knowledge stored in PLMs have factual errors, which impairs model performance on question answering or generation. Knowledge calibration aims to rectifie these wrong knowledge.  \nIn order to deal with the false facts, previous work focuses on complementing or modifying knowledge for a specific downstream task. Yao et al. (2022) proposed retrieving external knowledge during fine-tuning. Cao et al. (2021b) modified specific knowledge after finetuning. However, these methods do not generalize to multiple tasks. In this paper, we explore a task-agnostic method to directly calibrate general factual knowledge in PLMs without re-training from scratch. We aim to correct the false facts in PLMs. Since every single fact has multiple surfaces, we also expect that the calibrated knowledge should be generalizable to various text surfaces. Figure 1 illustrates the process of calibration. First, we detect the false knowledge in PLMs with a Contrastive Knowledge Assessing (CKA) method (demonstrated in Figure 2) . Since PLMs make black-box decisions, we evaluate PLMs via their predictions for simplification. The key motivation behind CKA is a plain argument that a PLM correctly learns a fact if and only if the model assigns the right fact higher scores than possible negative facts. For that false knowledge, we then propose CALINET to calibrate them by telling PLMs what the right fact is. Without compromising parameters in the original PLM, our approach calibrates the false knowledge by finetuning new parameters while the original parameters are fixed during calibration. Inspired","cbCair1gSE3luLu4","https://ap.wps.com/l/cbCair1gSE3luLu4","pdf",680578,11,"English","# Introduction\n## Contrastive Knowledge Assessment\n# CALINET Method\n## Experimental Evaluation\n## Knowledge Calibration Mechanism\n## Contributions","[{\"question\":\"What problem does this paper address about pretrained language models?\",\"answer\":\"Pretrained language models may store factual knowledge that is not always correct. The paper focuses on detecting and correcting false facts without retraining the whole model from scratch.\"},{\"question\":\"How does CALINET calibrate incorrect factual knowledge?\",\"answer\":\"CALINET uses a two-stage strategy: it first evaluates whether a PLM assigns higher scores to right facts than to plausible negative facts. Then it finetunes lightweight, newly added parameters for specific factual texts while keeping the original PLM parameters fixed.\"},{\"question\":\"How is the calibration effectiveness evaluated?\",\"answer\":\"Experiments are conducted on knowledge probing tasks and closed-book question answering, showing that calibrated models are more accurate and can generalize knowledge after fine-tuning.\"}]","Calibrating Factual Knowledge in Pretrained Language Models - Research Paper - December 2022 | PDF",28]