[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-203911-105":59,"doc-detail-203911-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","fine-tuned-transformers-show-clusters-of-similar-representations-across-layers-block-diagonal-cka-analysis","Fine-Tuned Transformers Show Clusters of Similar Representations Across Layers - Block Diagonal CKA Analysis","","Fine-tuning pretrained language encoders for downstream NLU succeeds, yet the internal changes after adaptation remain unclear. This study applies centered kernel alignment (CKA) to quantify representation similarity across layers in task-tuned models. Across twelve GLUE and additional tasks, fine-tuned RoBERTa and ALBERT display a consistent block diagonal similarity structure: strong within-cluster similarity in early and late layers, weak cross-cluster similarity. Results indicate later layers only marginally affect task performance, and they can be removed with minimal impact even without further tuning.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/fine-tuned-transformers-show-clusters-of-similar-representations-across-layers-block-diagonal-cka-analysis/203911/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/fine-tuned-transformers-show-clusters-of-similar-representations-across-layers-block-diagonal-cka-analysis/203911.png","ImageObject",300,407,{"name":92,"@type":93},"Emma Mercer","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-07","2026-09-04",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",12,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"How does the paper measure how representations change after fine-tuning?","Question",{"text":113,"@type":114},"It uses centered kernel alignment (CKA) to compute similarity scores between learned representations from different layers in untuned versus task-tuned models.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"What main pattern is found in the similarity of representations across layers?",{"text":118,"@type":114},"The experiments reveal a consistent block diagonal structure: early-layer and late-layer representations form distinct clusters with high within-cluster similarity and low between-cluster similarity.",{"name":120,"@type":111,"acceptedAnswer":121},"What does the similarity structure imply about later transformer layers for task performance?",{"text":122,"@type":114},"Because later layers remain highly similar across models, the paper concludes they only marginally contribute, and experiments show that discarding the top few layers has minimal effect even without additional tuning.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},203911,1788564873,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":52,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":130,"read_time":144},962084925502,"https://ap-avatar.wpscdn.com/davatar_6f874abed73319feea01a86fa6f0fab8","Fine-Tuned Transformers Show Clusters of Similar Representations Across Layers  \nJason Phang 1 , Haokun Liu2 , Samuel R. Bowman 134  \n1 Center for Data Science, New York University  \n2Dept. of Computer Science, University of North Carolina at Chapel Hill  \n3Dept. of Linguistics, New York University  \n4Dept. of Computer Science, New York University  \nCorrespondence: [jasonphang@nyu.edu](jasonphang@nyu.edu)  \nAbstract  \nDespite the success of fine-tuning pretrained language encoders like BERT for downstream natural language understanding (NLU) tasks, it is still poorly understood how neural networks change after fine-tuning. In this work, we use centered kernel alignment (CKA), a method for comparing learned representations, to measure the similarity of representations in task-tuned models across layers. In experiments across twelve NLU tasks, we discover a consistent block diagonal structure in the similarity of representations within fine-tuned RoBERTa and ALBERT models, with strong similarity within clusters of earlier and later layers, but not between them. The similarity of later layer representations implies that later layers only marginally contribute to task performance, and we verify in experiments that the top few layers of fine-tuned Transformers can be discarded without hurting performance, even with no further tuning.  \n1 Introduction  \nFine-tuning pretrained language encoders such as BERT (Devlin et al., 2019) and its successors (Liu et al., 2019b ; Lan et al., 2020 ; Clark et al., 2020 ; He et al., 2020) has proven to be highly successful, attaining state-of-the-art performance on many language tasks, but how do these models internally represent task-specific knowledge?  \nIn this work, we study how learned representations change through fine-tuning by studying the similarity of representations between layers of untuned and task-tuned models. We use centered kernel alignment (CKA; Kornblith et al., 2019) to measure representation similarity and conduct extensive experiments across three pretrained encoders and twelve language understanding tasks.  \nWe discover a consistent, block diagonal structure (Figure 1c,d) in the similarity of learned representations for almost all task-tuned RoBERTa  \nFT layers Orig layers  \n(a) ORIG ~~ ~~ ORIG  \n24  \n16  \n8  \n0  \nOrig layers  \n0 8 16 24 FT layers  \nFT layers FT layers  \n(b) FT ~~ ~~ ORIG  \nOrig layers  \n(d) FT[1] ~~ ~~ FT[2]  \n0 8 16 24 FT (run 2) layers  \n1.0  \n0.8  \n0.6  \n0.4  \n0.2  \n0.0  \nFigure 1: CKA similarity scores of CLS (classifier token) representations of ORIG (untuned ALBERT) and FT (fine-tuned) models on RTE, across different layers of the model. FT[1]–FT[2] compares two RTE models with different random restarts. ORIG–ORIG and FT– FT are symmetric by construction. Fine-tuned models exhibit a block-diagonal structure in the representation similarities. The same color scale is used in all plots.  \nand ALBERT models, where early layer representations and later layer representations form two distinct clusters, with high intra-cluster and low inter-cluster similarity.  \nGiven the strong representation similarity of later model layers, we hypothesize that many of the later layers only marginally contribute to task performance. We show in experiments that the later layers oftask-tuned RoBERTa and ALBERT can indeed be discarded with minimal impact to performance, even without any further fine-tuning.  \n2 Experimental Setup  \nModels For the majority of our experiments, we consider three commonly used language-encoding models: RoBERTa (Liu et al., 2019b), ALBERT (Lan et al., 2020) and ELECTRA (Clark et al., 2020) . Because of the large number of exper-  \n529  \nProceedings of the Fourth BlackboxNLP Workshop on Analyzing and Interpreting Neural Networks for NLP, pages 529–538  \nOnline, November 11, 2021 . ©2021 Association for Computational Linguistics  \niments being performed, we use RoBERTaBASE , ALBERT LARGEV2 and ELECTRA BASE rather than the largest available versions of these","cbCairN1xHfPU476","https://ap.wps.com/l/cbCairN1xHfPU476","pdf",1514856,"English","# Introduction\n# Experimental Setup\n## Models\n## Tasks\n## Optimization\n# Representation Similarity with CKA","[{\"question\":\"How does the paper measure how representations change after fine-tuning?\",\"answer\":\"It uses centered kernel alignment (CKA) to compute similarity scores between learned representations from different layers in untuned versus task-tuned models.\"},{\"question\":\"What main pattern is found in the similarity of representations across layers?\",\"answer\":\"The experiments reveal a consistent block diagonal structure: early-layer and late-layer representations form distinct clusters with high within-cluster similarity and low between-cluster similarity.\"},{\"question\":\"What does the similarity structure imply about later transformer layers for task performance?\",\"answer\":\"Because later layers remain highly similar across models, the paper concludes they only marginally contribute, and experiments show that discarding the top few layers has minimal effect even without additional tuning.\"}]","Fine-Tuned Transformers Show Clusters of Similar Representations Across Layers - Block Diagonal CKA Analysis | PDF",25]