[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-1-en-105":3,"doc-seo-337170-105":53,"doc-detail-337170-en":127},{"code":4,"msg":5,"data":6},0,"success",[7,14,19,24,29,34,39,44,49],{"id":8,"doc_module":9,"doc_module_name":10,"category_name":11,"show_sort_weight":12,"slug":13},11,1,"Template","Presentations",90,"presentations",{"id":15,"doc_module":9,"doc_module_name":10,"category_name":16,"show_sort_weight":17,"slug":18},12,"Resumes",80,"resumes",{"id":20,"doc_module":9,"doc_module_name":10,"category_name":21,"show_sort_weight":22,"slug":23},14,"Invoices",70,"invoices",{"id":25,"doc_module":9,"doc_module_name":10,"category_name":26,"show_sort_weight":27,"slug":28},15,"Posters",60,"posters",{"id":30,"doc_module":9,"doc_module_name":10,"category_name":31,"show_sort_weight":32,"slug":33},16,"Social Media",50,"social-media",{"id":35,"doc_module":9,"doc_module_name":10,"category_name":36,"show_sort_weight":37,"slug":38},17,"Forms",40,"forms",{"id":40,"doc_module":9,"doc_module_name":10,"category_name":41,"show_sort_weight":42,"slug":43},18,"Letters",30,"letters",{"id":45,"doc_module":9,"doc_module_name":10,"category_name":46,"show_sort_weight":47,"slug":48},21,"Paper Templates",5,"papers-templates",{"id":50,"doc_module":9,"doc_module_name":10,"category_name":51,"show_sort_weight":4,"slug":52},158,"General","general-158",{"code":4,"msg":54,"data":55},"ok",{"site_id":56,"language":57,"slug":58,"title":59,"keywords":60,"description":61,"schema_data":62,"social_meta":120,"head_meta":122,"extra_data":124,"updated_unix":126},105,"en","mixed-initiative-approach-to-extract-data-from-pictures-of-medical-invoice","Mixed-Initiative Approach to Extract Data from Pictures of Medical Invoice","","Extracting data from pictures of medical records is a common insurance workflow, yet full automation remains difficult due to low image quality and diverse invoice templates. This paper presents a mixed-initiative pipeline that combines deep-learning-based automatic prediction with task-specific heuristics mediated by a user. A user study with 12 participants shows the approach can reduce drawbacks of fully automated extraction while staying within acceptable completion time. Findings, limitations, and future directions for handling complicated tables are discussed.",{"@graph":63,"@context":119},[64,80,102],{"@type":65,"itemListElement":66},"BreadcrumbList",[67,71,74,77],{"item":68,"name":69,"@type":70,"position":9},"https://docshare.wps.com","Home","ListItem",{"item":72,"name":10,"@type":70,"position":73},"https://docshare.wps.com/template/",2,{"item":75,"name":11,"@type":70,"position":76},"https://docshare.wps.com/template/presentations/",3,{"item":78,"name":59,"@type":70,"position":79},"https://docshare.wps.com/template/mixed-initiative-approach-to-extract-data-from-pictures-of-medical-invoice/337170/",4,{"url":78,"name":59,"@type":81,"image":82,"author":87,"headline":59,"publisher":90,"fileFormat":93,"inLanguage":57,"description":61,"dateModified":94,"datePublished":95,"encodingFormat":93,"isAccessibleForFree":96,"interactionStatistic":97},"DigitalDocument",{"url":83,"@type":84,"width":85,"height":86},"https://docshare.wps.com/thumbnails/mixed-initiative-approach-to-extract-data-from-pictures-of-medical-invoice/337170.png","ImageObject",442,249,{"name":88,"@type":89},"Emma Mercer","Person",{"url":68,"name":91,"@type":92},"DocShare","Organization","application/pdf","2026-10-22","2026-09-21",true,{"@type":98,"interactionType":99,"userInteractionCount":101},"InteractionCounter",{"@type":100},"ViewAction",10,{"@type":103,"mainEntity":104},"FAQPage",[105,111,115],{"name":106,"@type":107,"acceptedAnswer":108},"Why is extracting data from medical invoice photos challenging to automate?","Question",{"text":109,"@type":110},"Automation is difficult because smartphone images often have low quality and the invoices use varied templates, which break assumptions used by traditional scanned/fax processing methods.","Answer",{"name":112,"@type":107,"acceptedAnswer":113},"What does the proposed mixed-initiative pipeline do?",{"text":114,"@type":110},"It first builds a fully automated pipeline to recognize medical invoices, then lets users inspect and correct errors at each stage using visualizations and interactions.",{"name":116,"@type":107,"acceptedAnswer":117},"What evidence supports the effectiveness of the approach?",{"text":118,"@type":110},"A user study with 12 participants indicates the interactive pipeline can capture about 97% of relations in invoice images with comparable completion time and less human effort.","https://schema.org",{"og:url":78,"og:type":121,"og:title":59,"og:site_name":91,"og:description":61},"article",{"robots":123,"canonical":78},"index,follow",{"doc_id":125,"site_id":56},337170,1791156874,{"code":4,"msg":5,"data":128},{"doc_id":125,"user_id":129,"nickname":88,"user_avatar":130,"doc_module":9,"category_id":8,"category_name":11,"doc_title":59,"doc_description":61,"doc_content":131,"file_id":132,"file_url":133,"file_type":134,"file_size":135,"view_count":101,"is_deleted":4,"is_public":9,"is_downloadable":9,"audit_status":9,"page_count":47,"language":136,"language_code":57,"site_id":56,"html_lang":57,"table_of_contents":137,"faqs":138,"seo_title":139,"seo_description":61,"update_tm":140,"read_time":73},962084925502,"https://ap-avatar.wpscdn.com/davatar_6f874abed73319feea01a86fa6f0fab8","Mixed-Initiative Approach to Extract Data from Pictures of Medical Invoice  \nSeokweon Jung Kiroong Choe Seokhyeon Park Hyung-Kwon Ko Youngtaek Kim * Jinwook Seo †  \nDepartment of Computer Science and Engineering, Seoul National University, Seoul, Republic of Korea  \nABSTRACT  \nExtracting data from pictures of medical records is a common task in the insurance industry as the patients often send their medical invoices taken by smartphone cameras. However, the overall process is still challenging to be fully automated because of low image quality and variation of templates that exist in the status quo. In this paper, we propose a mixed-initiative pipeline for extracting data from pictures of medical invoices, where deep-learning-based automatic prediction models and task-specific heuristics work together under the mediation of a user. In the user study with 12 participants, we confirmed our mixed-initiative approach can supplement the drawbacks of a fully automated approach within an acceptable completion time. We further discuss the findings, limitations, and future works for designing a mixed-initiative system to extract data from pictures of a complicated table.  \nIndex Terms: Human-centered computing—Human computer interaction—Interactive systems and tools—User interface toolkits;  \n1 INTRODUCTION  \nSupplier invoices are important data for insurance companies when they decide how much they should reimburse the customer for medical expenses. Accounts Payable teams are in charge of managing invoice data. They review, record, and back up those invoices transmitted to the companies [14] . Customers usually submit invoices in pictures taken using their smartphones, and thus extracting data from images is a crucial task for the team. The extracted data should be exact in amount and contain no errors.  \nAs manual transcription of the invoice is a very time-consuming and labor-intensive task, many researchers have tried to automate the procedure [3, 13, 14] . One of the most successful methods uses coordinates information explicitly. This method requires the image to be straight and aligned, and this is usually the case for scanned or faxed images. A growing number of customers, however, are submitting smartphone pictures of invoices. Since such pictures do not satisfy the key assumptions made for scanned or fax images, existing invoice processing methods are not feasible for extracting data from them.  \nIn this paper, we propose a mixed-initiative approach to extract data from pictures of medical invoices. First, we developed a fully automated pipeline that tackles the core difficulty in recognizing medical invoices, such as finding a data table from images and recognizing template variation. Then, we observed that the majority of errors contained in the pipeline could be easily corrected by human interactions. We developed a mixed-initiative pipeline, in which users can discover errors of each stage with visualizations and correct them with interactions. A user study involving 12 participants confirmed that our interactive pipeline can capture about 97% of relations in invoice images within comparable time and with less human efforts.  \nIn summary, our study makes the following contributions:  \n• We investigated the characteristics of the real-world dataset containing smartphone pictures of a medical invoice and defined core difficulties in recognizing them.  \n• We enhanced the existing table structure recognition algorithm with heuristics and devised the fully automated pipeline.  \n• We introduce the mixed-initiative approach as an alternative solution to the problem by inviting users to correct intermediate machine output at a low cost.  \n* e-mail: fswjung, krchoe, shpark, hkko, [ytaek.kim](ytaek.kimg@hcil.snu.ac.kr)[g](ytaek.kimg@hcil.snu.ac.kr)[@hcil.snu.ac.kr](ytaek.kimg@hcil.snu.ac.kr)[ ](ytaek.kimg@hcil.snu.ac.kr)†e-mail: [jseo@snu.ac.kr](jseo@snu.ac.kr)  \n2 RELATED WORK  \nWhen automating invoice processing, invoice recognition shou","cbCaid3rO98Q85kb","https://ap.wps.com/l/cbCaid3rO98Q85kb","pdf",7192785,"English","# Abstract\n# 1 Introduction\n# 2 Related Work","[{\"question\":\"Why is extracting data from medical invoice photos challenging to automate?\",\"answer\":\"Automation is difficult because smartphone images often have low quality and the invoices use varied templates, which break assumptions used by traditional scanned/fax processing methods.\"},{\"question\":\"What does the proposed mixed-initiative pipeline do?\",\"answer\":\"It first builds a fully automated pipeline to recognize medical invoices, then lets users inspect and correct errors at each stage using visualizations and interactions.\"},{\"question\":\"What evidence supports the effectiveness of the approach?\",\"answer\":\"A user study with 12 participants indicates the interactive pipeline can capture about 97% of relations in invoice images with comparable completion time and less human effort.\"}]","Mixed-Initiative Approach to Extract Data from Pictures of Medical Invoice | PDF",1790018003]