[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-141028-105":3,"detail-sidebar-cat-0-en-105":75,"doc-detail-141028-en":125},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":68,"head_meta":70,"extra_data":72,"updated_unix":74},105,"en","a-video-is-worth-4096-tokens-verbalize-videos-to-understand-them-in-zero-shot","A Video Is Worth 4096 Tokens - Verbalize Videos To Understand Them In Zero Shot","","Multimedia content such as advertisements and story videos blends text, visuals, audio, and narrative devices like emotions and slogans to communicate meaning. Large annotated datasets for multimedia understanding remain scarce, limiting supervised models for real-world performance. Large language models show strong zero-shot NLP capabilities, prompting a new approach: verbalize long videos into natural-language descriptions, then run video-understanding tasks on the generated story. Experiments across fifteen tasks show clear gains over supervised baselines, and a new persuasion-strategy dataset is released.",{"@graph":14,"@context":67},[15,34,50],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/a-video-is-worth-4096-tokens-verbalize-videos-to-understand-them-in-zero-shot/141028/",4,{"url":32,"name":10,"@type":35,"author":36,"headline":10,"publisher":39,"fileFormat":42,"inLanguage":8,"description":12,"dateModified":43,"datePublished":44,"encodingFormat":42,"isAccessibleForFree":45,"interactionStatistic":46},"DigitalDocument",{"name":37,"@type":38},"Mason","Person",{"url":19,"name":40,"@type":41},"DocShare","Organization","application/pdf","2026-09-11","2026-08-25",true,{"@type":47,"interactionType":48,"userInteractionCount":30},"InteractionCounter",{"@type":49},"ViewAction",{"@type":51,"mainEntity":52},"FAQPage",[53,59,63],{"name":54,"@type":55,"acceptedAnswer":56},"What problem does the paper address in multimedia video understanding?","Question",{"text":57,"@type":58},"It targets the lack of large annotated training datasets for multimedia, especially for long and complex story videos, which prevents supervised models from achieving satisfactory real-world performance.","Answer",{"name":60,"@type":55,"acceptedAnswer":61},"How does the proposed method improve video understanding without training task-specific models?",{"text":62,"@type":58},"It verbalizes long videos into natural-language descriptions and performs downstream video-understanding tasks on the generated story, yielding zero-shot performance.",{"name":64,"@type":55,"acceptedAnswer":65},"What evidence supports the effectiveness of the method?",{"text":66,"@type":58},"Extensive experiments on fifteen video-understanding tasks show significantly better results than supervised baselines, even though the approach is zero-shot.","https://schema.org",{"og:url":32,"og:type":69,"og:title":10,"og:site_name":40,"og:description":12},"article",{"robots":71,"canonical":32},"index,follow",{"doc_id":73,"site_id":7},141028,1787647868,{"code":4,"msg":76,"data":77},"success",[78,82,86,90,95,100,105,109,114,117,121],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":79,"show_sort_weight":80,"slug":81},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":83,"show_sort_weight":84,"slug":85},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":87,"show_sort_weight":88,"slug":89},"Exam",70,"exam",{"id":91,"doc_module":4,"doc_module_name":25,"category_name":92,"show_sort_weight":93,"slug":94},5,"Comic",60,"comic",{"id":96,"doc_module":4,"doc_module_name":25,"category_name":97,"show_sort_weight":98,"slug":99},6,"Technology",50,"technology",{"id":101,"doc_module":4,"doc_module_name":25,"category_name":102,"show_sort_weight":103,"slug":104},7,"Healthcare",40,"healthcare",{"id":106,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":107,"slug":108},8,30,"research-report",{"id":110,"doc_module":4,"doc_module_name":25,"category_name":111,"show_sort_weight":112,"slug":113},9,"Religion & Spirituality",20,"religion-spirituality",{"id":112,"doc_module":4,"doc_module_name":25,"category_name":115,"show_sort_weight":112,"slug":116},"World Cup","world-cup",{"id":118,"doc_module":4,"doc_module_name":25,"category_name":119,"show_sort_weight":118,"slug":120},10,"Lifestyle","lifestyle",{"id":122,"doc_module":4,"doc_module_name":25,"category_name":123,"show_sort_weight":91,"slug":124},19,"General","general",{"code":4,"msg":76,"data":126},{"doc_id":73,"user_id":127,"nickname":37,"user_avatar":128,"doc_module":4,"category_id":106,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":129,"file_id":130,"file_url":131,"file_type":132,"file_size":133,"view_count":30,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":134,"language":135,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":136,"faqs":137,"seo_title":138,"seo_description":12,"update_tm":74,"read_time":139},5909887256941,"https://ap-avatar.wpscdn.com/davatar_9964176cb1d06d4a9deccf72a44ae3dc","A Video Is Worth 4096 Tokens:  \nVerbalize Videos To Understand Them In Zero Shot  \nAanisha Bhattacharyya⋆ Yaman K Singla⋆  Balaji Krishnamurthy Rajiv Ratn Shah Changyou Chen  \nAdobe Media and Data Science Research (MDSR), IIIT-Delhi, State University of New York at Buffalo  \narXiv :2305 .09758v3 [ cs .CV] 26 Oct 2023  \nAbstract  \nMultimedia content, such as advertisements and story videos, exhibit a rich blend of creativity and multiple modalities. They incorporate elements like text, visuals, audio, and storytelling techniques, employing devices like emotions, symbolism, and slogans to convey meaning. There is a dearth of large annotated training datasets in the multimedia domain hindering the development of supervised learning models with satisfactory performance for real-world applications. On the other hand, the rise of large language models (LLMs) has witnessed remarkable zero-shot performance in various natural language processing (NLP) tasks, such as emotion classification, questionanswering, and topic classification. To leverage such advanced techniques to bridge this performance gap in multimedia understanding, we propose verbalizing long videos to generate their descriptions in natural language, followed by performing video-understanding tasks on the generated story as opposed to the original video. Through extensive experiments on fifteen video-understanding tasks, we demonstrate that our method, despite being zero-shot, achieves significantly better results than supervised baselines for video understanding. Furthermore, to alleviate a lack of story understanding benchmarks, we publicly release the first dataset on a crucial task in computational social science on persuasion strategy identification.  \n1 Introduction  \n“We are, as a species, addicted to stories. Even when the body goes to sleep, the mind stays up all night, telling itself stories.” -Jonathan Gottschall  \nMost videos we encounter in the day-to-day, like movies, documentaries, advertisements, and usergenerated content like Tiktok and Youtube shorts,  \n⋆Equal Contribution. Contact ykumar@adobe.com for questions and suggestions.  \ndepict some form of a story. Despite this, most work in the multimedia understanding domain has been about simple videos containing a single action or photo streams (Li et al., 2020) . Beyond understanding objects, actions, and scenes lies interpreting causal structure, making sense of visual, textual, and audio input to tie disparate moments together as they give rise to a cohesive narrative of events through time. This requires moving from reasoning about single activity and static moments to sequences of images and audio that depict events as they occur and change. Progressing from singleaction videos to story videos allows us to begin to reason about complex cognitive tasks like emotions depicted and persuasion strategies used.  \nRecently, large video pre-trained models (LVMs) like VideoMAE (Tong et al., 2022), InternVideo (Wang et al., 2022), and VideoCLIP (Xu et al., 2021) have proved to be powerful in enhancing reasoning skills on video data. For e.g., InternVideo showed a performance increase in action classification and question answering tasks. Nevertheless, these models have a few shortcomings that impair their performance on video understanding tasks: 1) LVMs are mostly trained on short videos (\u003C10s) consisting of majorly motion-centric actions, such as those present in Kinetics (Kay et al., 2017) and Something Something v2 (Goyal et al., 2017); and 2) they often require a significant amount of task-specific finetuning data to perform on videounderstanding tasks like summarization, questionanswering, and emotion classification. On the other hand, stories are often longer than 10 seconds and are typically much more complex than motioncentric videos. For example, we test our models on five datasets with average video lengths of 12.4 minutes (video story) and 3.5 minutes (other tasks) . Our videos contain dialogues, te","cbCair50JeLSXUip","https://ap.wps.com/l/cbCair50JeLSXUip","pdf",4009774,18,"English","# Introduction\n## Motivation for story video understanding\n## Limitations of large video pre-trained models\n## Proposed verbalization pipeline","[{\"question\":\"What problem does the paper address in multimedia video understanding?\",\"answer\":\"It targets the lack of large annotated training datasets for multimedia, especially for long and complex story videos, which prevents supervised models from achieving satisfactory real-world performance.\"},{\"question\":\"How does the proposed method improve video understanding without training task-specific models?\",\"answer\":\"It verbalizes long videos into natural-language descriptions and performs downstream video-understanding tasks on the generated story, yielding zero-shot performance.\"},{\"question\":\"What evidence supports the effectiveness of the method?\",\"answer\":\"Extensive experiments on fifteen video-understanding tasks show significantly better results than supervised baselines, even though the approach is zero-shot.\"}]","A Video Is Worth 4096 Tokens - Verbalize Videos To Understand Them In Zero Shot | PDF",45]