[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-280410-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-280410-en":131},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","ikiwisi-an-interactive-visual-pattern-generator-for-evaluating-the-reliability-of-vision-language-models-without-ground-truth","IKIWISI - An Interactive Visual Pattern Generator for Evaluating the Reliability of Vision-Language Models Without Ground Truth","","IKIWISI (“I Know It When I See It”) introduces an interactive visual pattern generator to assess vision-language models on video object recognition when ground truth is unavailable. It converts model outputs into a binary heatmap of green object presence and red object absence, leveraging human pattern recognition for fast reliability audits. The method includes “spy objects” to detect hallucinations on known-absent items and compares user judgments with objective metrics when available.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/ikiwisi-an-interactive-visual-pattern-generator-for-evaluating-the-reliability-of-vision-language-models-without-ground-truth/280410/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/ikiwisi-an-interactive-visual-pattern-generator-for-evaluating-the-reliability-of-vision-language-models-without-ground-truth/280410.png","ImageObject",300,407,{"name":42,"@type":43},"Violet","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-11-04","2026-09-16",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",15,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What problem does IKIWISI address in vision-language video recognition?","Question",{"text":63,"@type":64},"IKIWISI targets reliability assessment when ground truth is unavailable, making it hard for users to verify whether predicted objects exist in video frames in real time.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How does IKIWISI visualize model reliability without ground truth?",{"text":68,"@type":64},"It transforms outputs into a binary heatmap where green indicates object presence and red indicates object absence, enabling users to judge reliability by inspecting visual patterns.",{"name":70,"@type":61,"acceptedAnswer":71},"What are “spy objects” and why are they used?",{"text":72,"@type":64},"“Spy objects” are adversarial instances that users know are absent; they help reveal hallucinations when models produce detections for nonexistent items.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},280410,1789547873,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,111,115,120,123,127],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":108,"show_sort_weight":109,"slug":110},7,"Healthcare",40,"healthcare",{"id":112,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":113,"slug":114},8,30,"research-report",{"id":116,"doc_module":4,"doc_module_name":25,"category_name":117,"show_sort_weight":118,"slug":119},9,"Religion & Spirituality",20,"religion-spirituality",{"id":118,"doc_module":4,"doc_module_name":25,"category_name":121,"show_sort_weight":118,"slug":122},"World Cup","world-cup",{"id":124,"doc_module":4,"doc_module_name":25,"category_name":125,"show_sort_weight":124,"slug":126},10,"Lifestyle","lifestyle",{"id":128,"doc_module":4,"doc_module_name":25,"category_name":129,"show_sort_weight":97,"slug":130},19,"General","general",{"code":4,"msg":82,"data":132},{"doc_id":79,"user_id":133,"nickname":42,"user_avatar":134,"doc_module":4,"category_id":112,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":140,"language":141,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":12,"update_tm":80,"read_time":145,"preview_page_count":4},4398048950312,"https://ap-avatar.wpscdn.com/avatar/400002538284de19e3c?_k=1778320343897328908","arXiv :2505 .22305v1 [ cs .CV] 28 May 2025  \nIKIWISI: An Interactive Visual Pattern Generator for Evaluating the Reliability of Vision-Language Models Without Ground Truth  \nMd Touhidul Islam  \nPennsylvania State University University Park, PA, USA [touhid@psu.edu](touhid@psu.edu)  \nImran Kabir  \nPennsylvania State University University Park, PA, USA [ibk5106@psu.edu](ibk5106@psu.edu)  \nMd Alimoor Reza  \nDrake University Des Moines, IA, USA [md.reza@drake.edu](md.reza@drake.edu)  \nSyed Masum Billah  \nPennsylvania State University University Park, PA, United States [sbillah@psu.edu](sbillah@psu.edu)  \nFigure 1: IKIWISI’s interactive interface for evaluating vision-language models (best viewed in color). The interface shows model and video selection options (A-C), video keyframes (D), object selection panel (E), and the core binary heat map (F), where green and red cells create visual patterns that help users assess model reliability. All the components and their roles are described in detail in Sec. 3.1.  \nABSTRACT  \nWe present IKIWISI (\"I Know It When I See It\"), an interactive visual pattern generator for assessing vision-language models in video object recognition when ground truth is unavailable. IKIWISI transforms model outputs into a binary heatmap where green  \nPermission to make digital or hard copies of all or part of this work for personal or classroom use is granted without fee provided that copies are not made or distributed for profit or commercial advantage and that copies bear this notice and the full citation on the first page. Copyrights for components of this work owned by others than the author(s) must be honored. Abstracting with credit is permitted. To copy otherwise, or republish, to post on servers or to redistribute to lists, requires prior specific permission [and/or a fee. Request permissions from permissions@acm.org](and/or a fee. Request permissions from permissions@acm.org).  \nDIS ’25, July 5–9, 2025, Funchal, Portugal  \n© 2025 Copyright held by the owner/author(s) . Publication rights licensed to ACM. ACM ISBN 979-8-4007-1485-6/2025/07. . . $15.00 [https://doi.org/10.1145/3715336.3735754](https://doi.org/10.1145/3715336.3735754)  \ncells indicate object presence and red cells indicate object absence. This visualization leverages humans’ innate pattern recognition abilities to evaluate model reliability. IKIWISI introduces \"spy objects\"—adversarial instances users know are absent—to discern models hallucinating on nonexistent items. The tool functions as a cognitive audit mechanism, surfacing mismatches between human and machine perception by visualizing where models diverge from human understanding.  \nOur study with 15 participants found that users considered IKIWISI easy to use, made assessments that correlated with objective metrics when available, and reached informed conclusions by examining only a small fraction of heatmap cells. This approach not only complements traditional evaluation methods through visual assessment of model behavior with custom object sets, but also reveals  \nopportunities for improving alignment between human perception and machine understanding in vision-language systems.  \nCCS CONCEPTS  \n• Human-centered computing → Visualization design and evaluation methods; Participatory design; • Computing methodologies → Computer vision.  \nKEYWORDS  \nLarge Multi-Modal Models (LMMs), Subjective Evaluation, Open Vocabulary Model, F1-Score; Research-through-Design, cognitive auditing tool; visual perception  \nACM Reference Format:  \nMd Touhidul Islam, Imran Kabir, Md Alimoor Reza, and Syed Masum Billah.  \n2025. IKIWISI: An Interactive Visual Pattern Generator for Evaluating the Reliability of Vision-Language Models Without Ground Truth. In Designing Interactive Systems Conference (DIS ’25), July 5–9, 2025, Funchal, Portugal. ACM, New York, NY, USA, 21 pages. [https://doi.org/10.1145/3715336.3735754](https://doi.org/10.1145/3715336.3735754)  \n1 INTRODUCTION  \nHuman visual perception is","cbCaivbvCSUw9aE1","https://ap.wps.com/l/cbCaivbvCSUw9aE1","pdf",11184155,21,"English","# Abstract\n# Introduction\n# Background and Motivation\n# Related Challenges","[{\"question\":\"What problem does IKIWISI address in vision-language video recognition?\",\"answer\":\"IKIWISI targets reliability assessment when ground truth is unavailable, making it hard for users to verify whether predicted objects exist in video frames in real time.\"},{\"question\":\"How does IKIWISI visualize model reliability without ground truth?\",\"answer\":\"It transforms outputs into a binary heatmap where green indicates object presence and red indicates object absence, enabling users to judge reliability by inspecting visual patterns.\"},{\"question\":\"What are “spy objects” and why are they used?\",\"answer\":\"“Spy objects” are adversarial instances that users know are absent; they help reveal hallucinations when models produce detections for nonexistent items.\"}]","IKIWISI - An Interactive Visual Pattern Generator for Evaluating the Reliability of Vision-Language Models Without Ground Truth | PDF",53]