[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-328002-105":59,"doc-detail-328002-en":135},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":128,"head_meta":130,"extra_data":132,"updated_unix":134},105,"en","do-all-visual-tokens-matter-equally-object-evidence-preserving-token-merging-for-vision-language-retrieval-328002","Do All Visual Tokens Matter Equally? - Object-Evidence Preserving Token Merging for Vision-Language Retrieval","","Multi-vector vision-language retrieval preserves fine-grained visual evidence via maximum-similarity late interaction, yet dense image tokens make storage and scoring costly. Existing token compression can erase or collapse object- and region-level evidence needed by future query tokens. SaMer proposes object-aware token merging that compresses image-side post-projector tokens into K centroids while keeping the late-interaction interface unchanged. Using object annotations only as a training merge prior and adapting only the shared projection layer, SaMer removes over 93% of tokens (K=64), reduces ColPali storage by 16.09×, and improves R@1 on Flickr30K and MSCOCO with stronger phrase-level grounding.",{"@graph":69,"@context":127},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/do-all-visual-tokens-matter-equally-object-evidence-preserving-token-merging-for-vision-language-retrieval-328002/328002/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/do-all-visual-tokens-matter-equally-object-evidence-preserving-token-merging-for-vision-language-retrieval-328002/328002.png","ImageObject",300,407,{"name":92,"@type":93},"Rowan","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-29","2026-09-21",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",14,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119,123],{"name":110,"@type":111,"acceptedAnswer":112},"What problem does SaMer address in multi-vector vision-language retrieval?","Question",{"text":113,"@type":114},"Dense image-side tokens make storage and MaxSim scoring expensive, but naive compression can remove or merge away object- and region-level evidence that query tokens need later.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"How does SaMer preserve object-level evidence during token merging?",{"text":118,"@type":114},"SaMer performs object-aware merging that compresses post-projector tokens into K representative centroids while preserving the original late-interaction interface, using object annotations as a merge prior during training to discourage cross-instance mixing.",{"name":120,"@type":111,"acceptedAnswer":121},"Does SaMer require object detectors or bounding boxes at inference time?",{"text":122,"@type":114},"No. SaMer performs annotation-free feature-spatial merging at inference and does not rely on ground-truth bounding boxes or object detectors.",{"name":124,"@type":111,"acceptedAnswer":125},"What improvements does SaMer achieve with K=64?",{"text":126,"@type":114},"It removes more than 93% of image-side tokens, reduces ColPali storage by 16.09×, and improves R@1 on Flickr30K and MSCOCO, supported by stronger phrase-level grounding.","https://schema.org",{"og:url":83,"og:type":129,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":131,"canonical":83},"index,follow",{"doc_id":133,"site_id":62},328002,1790125416,{"code":4,"msg":5,"data":136},{"doc_id":133,"user_id":137,"nickname":92,"user_avatar":138,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":139,"file_id":140,"file_url":141,"file_type":142,"file_size":143,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":105,"language":144,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":145,"faqs":146,"seo_title":147,"seo_description":67,"update_tm":148,"read_time":149,"preview_page_count":4},1099514067415,"https://ap-avatar.wpscdn.com/avatar/100002539d78ffe74a7?x-image-process=image/resize,m_fixed,w_180,h_180&k=1779092875211072502","Do All Visual Tokens Matter Equally?  \nObject-Evidence Preserving Token Merging for Vision-Language Retrieval  \nSuhyeong Park 1,3 , Junha Jung2,3 , Jungwoo Park2,3 , Jaewoo Kang2,3∗  \n1The Catholic University of Korea  \n2 Korea University  \n3AIGEN Sciences Inc.  \n[pshpulip22@catholic.ac.kr](pshpulip22@catholic.ac.kr)  \n{goodjungjun,jungwoo-park,[kangj}@korea.ac.kr](kangj}@korea.ac.kr)  \narXiv :2607 .04605v 1 [ cs .IR] 6 Jul 2026  \nAbstract  \nMulti-vector vision-language retrieval preserves fine-grained visual evidence through maximum-similarity late interaction, but dense image-side tokens make storage and scoring expensive. Existing token compression methods reduce this cost, yet they can remove or collapse object-and region-level evidence that future query tokens may need to select. We propose SaMer, an object-aware token merging framework that compresses image-side post-projector tokens into K representative centroids while preserving the original late-interaction interface. SaMer uses object annotations only during training as a merge prior to discourage cross-instance mixing, requires no ground-truth bounding boxes or detectors at inference time, and adapts only the shared projection layer with frozen vision and language backbones. With K = 64, SaMer removes more than 93% of image-side tokens and reduces ColPali storage by  \n16.09 ×, while improving R@1 on Flickr30K and MSCOCO. These gains arise because object-aware merging preserves query-selectable object evidence that pruning or feature-only pooling can remove or collapse. SaMer also outperforms compression baselines and shows stronger phrase-level grounding, suggesting that efficient multi-vector retrieval depends not only on reducing token count, but on preserving the evidence future query tokens need to select.  \nCode—[https://github.com/dmis-lab/SaMer](https://github.com/dmis-lab/SaMer)  \n1 Introduction  \nEfficient multi-vector vision-language retrieval requires preserving the object-, attribute-, and relation-level evidence that query tokens can select under late interaction. Multivector retrieval has been widely adopted in text retrieval since ColBERT (Khattab and Zaharia 2020; Santhanam et al. 2022b), which performs maximum-similarity (MaxSim) matching between contextualized query and document token embeddings. Recent vision-language retrievers such as ColPali (Faysse et al. 2024) extend this paradigm to visual inputs by storing image-side patch embeddings and comparing them with query tokens through MaxSim. Similarly, ColQwen2 (Faysse et al. 2024) builds on Qwen2-VL (Wanget al. 2024), whose visual tokenization supports flexible image representation. These models use MaxSim to compare each query token with all image-side tokens and select the  \n∗Corresponding author.  \nmost similar visual token as evidence, making retrieval sensitive to objects, attributes, and relations rather than only global image-text similarity (Li et al. 2019; Diao et al. 2021) .  \nHowever, preserving token-level image representations makes retrieval expensive in both storage and scoring. A multi-vector retriever stores hundreds to over a thousand image-side token embeddings per image, and retrieval requires MaxSim comparisons between query tokens and all stored image tokens. As the index grows, both memory footprint and scoring latency increase rapidly, limiting largescale image search and retrieval-augmented visual question answering. The issue is especially pronounced in patch-based visual representations, where each image produces many local tokens that must be stored and compared during MaxSim scoring, making patch-level embeddings a major bottleneck for multi-vector VLM retrievers (Dosovitskiy et al. 2021; Tang et al. 2022; Ma et al. 2025a) .  \nOne strategy is to compress visual tokens through pruning or merging (Ryoo et al. 2021; Rao et al. 2021; Lianget al. 2022; Marin et al. 2023; Bolya et al. 2023) . Recent multimodal acceleration methods reduce visual tokens for large vision-lan","cbCaim82ACNxLjuZ","https://ap.wps.com/l/cbCaim82ACNxLjuZ","pdf",19163607,"English","# Abstract\n# Introduction\n## Background: MaxSim multi-vector retrieval\n## Challenge: expensive storage and scoring\n## Token compression limitations\n## Proposed method: SaMer","[{\"question\":\"What problem does SaMer address in multi-vector vision-language retrieval?\",\"answer\":\"Dense image-side tokens make storage and MaxSim scoring expensive, but naive compression can remove or merge away object- and region-level evidence that query tokens need later.\"},{\"question\":\"How does SaMer preserve object-level evidence during token merging?\",\"answer\":\"SaMer performs object-aware merging that compresses post-projector tokens into K representative centroids while preserving the original late-interaction interface, using object annotations as a merge prior during training to discourage cross-instance mixing.\"},{\"question\":\"Does SaMer require object detectors or bounding boxes at inference time?\",\"answer\":\"No. SaMer performs annotation-free feature-spatial merging at inference and does not rely on ground-truth bounding boxes or object detectors.\"},{\"question\":\"What improvements does SaMer achieve with K=64?\",\"answer\":\"It removes more than 93% of image-side tokens, reduces ColPali storage by 16.09×, and improves R@1 on Flickr30K and MSCOCO, supported by stronger phrase-level grounding.\"}]","Do All Visual Tokens Matter Equally? - Object-Evidence Preserving Token Merging for Vision-Language Retrieval | PDF",1789978472,35]