[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-203719-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-203719-en":131},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","reliability-of-cka-as-a-similarity-measure-in-deep-learning-research-paper","RELIABILITY OF CKA AS A SIMILARITY MEASURE IN DEEP LEARNING - Research paper","","Comparing learned neural representations is a difficult yet essential task in deep learning. Centered Kernel Alignment (CKA), especially its linear variant, is widely used to compare representations across layers, architectures, and training initializations, but prior conclusions vary and known outlier effects remain insufficiently explained. This work formalizes CKA sensitivity to many simple transformations common in modern learning, clarifies sensitivity to outliers and to transformations that preserve linear separability, and experimentally exposes cases yielding unexpected results.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/reliability-of-cka-as-a-similarity-measure-in-deep-learning-research-paper/203719/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/reliability-of-cka-as-a-similarity-measure-in-deep-learning-research-paper/203719.png","ImageObject",300,407,{"name":42,"@type":43},"Kyle","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-10-09","2026-09-04",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",12,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What problem does the paper address regarding CKA in deep learning?","Question",{"text":63,"@type":64},"It studies how reliable Centered Kernel Alignment (CKA) is as a similarity measure, and identifies situations where CKA gives unexpected or counter-intuitive results.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How does the paper explain CKA sensitivity?",{"text":68,"@type":64},"It formally characterizes CKA sensitivity to a large class of simple transformations that commonly occur in ANNs, including transformations that preserve linear separability.",{"name":70,"@type":61,"acceptedAnswer":71},"Can CKA values be changed without affecting model behavior?",{"text":72,"@type":64},"Yes. The paper presents approaches for modifying representations so that functional behavior is maintained while the CKA value is substantially altered.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},203719,1788562834,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,111,115,120,123,127],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":108,"show_sort_weight":109,"slug":110},7,"Healthcare",40,"healthcare",{"id":112,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":113,"slug":114},8,30,"research-report",{"id":116,"doc_module":4,"doc_module_name":25,"category_name":117,"show_sort_weight":118,"slug":119},9,"Religion & Spirituality",20,"religion-spirituality",{"id":118,"doc_module":4,"doc_module_name":25,"category_name":121,"show_sort_weight":118,"slug":122},"World Cup","world-cup",{"id":124,"doc_module":4,"doc_module_name":25,"category_name":125,"show_sort_weight":124,"slug":126},10,"Lifestyle","lifestyle",{"id":128,"doc_module":4,"doc_module_name":25,"category_name":129,"show_sort_weight":97,"slug":130},19,"General","general",{"code":4,"msg":82,"data":132},{"doc_id":79,"user_id":133,"nickname":42,"user_avatar":134,"doc_module":4,"category_id":112,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":140,"language":141,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":12,"update_tm":80,"read_time":145},3985741905716,"https://ap-avatar.wpscdn.com/davatar_994ba38a5ba835b3df7d355c54d3ed8d","arXiv :2210 . 16156v2 [ cs .LG] 16 Nov 2022  \nRELIABILITY OF CKA AS A SIMILARITY MEASURE IN DEEP LEARNING  \nMohammadReza Davari 1 ;3 􀀃 Stefan Horoi 2 ;3 􀀃 Amine Natik 2 ;3 Guillaume Lajoie 2 ;3 Guy Wolf 2 ;3 y Eugene Belilovsky 1 ;3 y  \n1 Concordia University 2 Université de Montréal 3 Mila – Quebec AI Institute {mohammadreza.davari, [eugene.belilovsky}@concordia.ca](eugene.belilovsky}@concordia.ca)[ ](eugene.belilovsky}@concordia.ca){stefan.horoi, amine.natik, guillaume.lajoie, [guy.wolf}@umontreal.ca](guy.wolf}@umontreal.ca)  \nABSTRACT  \nComparing learned neural representations in neural networks is a challenging but important problem, which has been approached in different ways. The Centered Kernel Alignment (CKA) similarity metric, particularly its linear variant, has recently become a popular approach and has been widely used to compare representations of a network's different layers, of architecturally similar networks trained differently, or of models with different architectures trained on the same data. A wide variety of conclusions about similarity and dissimilarity of these various representations have been made using CKA. In this work we present analysis that formally characterizes CKA sensitivity to a large class of simple transformations, which can naturally occur in the context of modern machine learning. This provides a concrete explanation of CKA sensitivity to outliers, which has been observed in past works, and to transformations that preserve the linear separability of the data, an important generalization attribute. We empirically investigate several weaknesses of the CKA similarity metric, demonstrating situations in which it gives unexpected or counter-intuitive results. Finally we study approaches for modifying representations to maintain functional behaviour while changing the CKA value. Our results illustrate that, in many cases, the CKA value can be easily manipulated without substantial changes to the functional behaviour of the models, and call for caution when leveraging activation alignment metrics.  \n1 INTRODUCTION  \nIn the last decade, increasingly complex deep learning models have dominated machine learning and have helped us solve, with remarkable accuracy, a multitude of tasks across a wide array of domains. Due to the size and flexibility of these models it has been challenging to study and understand exactly how they solve the tasks we use them on. A helpful framework for thinking about these models is that of representation learning, where we view artificial neural networks (ANNs) as learning increasingly complex internal representations as we go deeper through their layers. In practice, it is often of interest to analyze and compare the representations of multiple ANNs. However, the typical high dimensionality of ANN internal representation spaces makes this a fundamentally difficult task.  \nTo address this problem, the machine learning community has tried finding meaningful ways to compare ANN internal representations and various representation (dis)similarity measures have been proposed (Li et al., 2015; Wang et al., 2018; Raghu et al., 2017; Morcos et al., 2018) . Recently, Centered Kernel Alignment (CKA) (Kornblith et al., 2019) was proposed and shown to be able to reliably identify correspondences between representations in architecturally similar networks trained on the same dataset but from different initializations, unlike past methods such as linear regression or CCA based methods (Raghu et al., 2017; Morcos et al., 2018) . While CKA can capture different notions of similarity between points in representation space by using different kernel functions, it was empirically shown in the original work that there are no real benefits to using CKA with a nonlinear kernel over its linear counterpart. As a result, linear CKA has been the preferred representation similarity measure of the machine learning community in recent years and other similarity measures (including nonlinear CKA) are sel","cbCaihu1164r8OMW","https://ap.wps.com/l/cbCaihu1164r8OMW","pdf",1395534,28,"English","# Abstract\n# Introduction\n## Background on representation similarity\n# Background on \n## CKA and linear CKA usage","[{\"question\":\"What problem does the paper address regarding CKA in deep learning?\",\"answer\":\"It studies how reliable Centered Kernel Alignment (CKA) is as a similarity measure, and identifies situations where CKA gives unexpected or counter-intuitive results.\"},{\"question\":\"How does the paper explain CKA sensitivity?\",\"answer\":\"It formally characterizes CKA sensitivity to a large class of simple transformations that commonly occur in ANNs, including transformations that preserve linear separability.\"},{\"question\":\"Can CKA values be changed without affecting model behavior?\",\"answer\":\"Yes. The paper presents approaches for modifying representations so that functional behavior is maintained while the CKA value is substantially altered.\"}]","RELIABILITY OF CKA AS A SIMILARITY MEASURE IN DEEP LEARNING - Research paper | PDF",71]