[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-detail-124961-en":3,"doc-seo-124961-105":30,"detail-sidebar-cat-0-en-105":91},{"code":4,"msg":5,"data":6},0,"success",{"doc_id":7,"user_id":8,"nickname":9,"user_avatar":10,"doc_module":4,"category_id":11,"category_name":12,"doc_title":13,"doc_description":14,"doc_content":15,"file_id":16,"file_url":17,"file_type":18,"file_size":19,"view_count":4,"is_deleted":4,"is_public":20,"is_downloadable":20,"audit_status":20,"page_count":21,"language":22,"language_code":23,"site_id":24,"html_lang":23,"table_of_contents":25,"faqs":26,"seo_title":27,"seo_description":14,"update_tm":28,"read_time":29},124961,2336464648322,"Aria","https://ap-avatar.wpscdn.com/avatar/2200025388227c56fec?_k=1778556882303663488",8,"Research & Report","Trustworthy Machine Learning under Imperfect Data - Data Collection, Robust Learning, and Real-World Considerations","Trustworthy machine learning requires evaluation protocols that remain reliable when data are imperfect, especially when labels are noisy or data collection is biased. This dissertation studies credible sample elicitation mechanisms, including multi-sample elicitation strategies, and applies them to text data evaluation such as LLM hallucination factualness using weighted models. It further develops robust learning methods for label noise, including f-divergence-based objectives and generalized label smoothing, and validates them on real-world human-annotation datasets like CIFAR-10N and CIFAR-100N. Additional work addresses multiple noisy labels and prior shift challenges, supported by theoretical analyses and experiments.","UC Santa Cruz  \nUC Santa Cruz Electronic Theses and Dissertations  \nTitle  \nTrustworthy Machine Learning under Imperfect Data: Data Collection, Robust Learning, and Real-World Considerations  \nPermalink  \n[https://escholarship.org/uc/item/44z3x2n5](https://escholarship.org/uc/item/44z3x2n5)  \nAuthor  \nWei, Jiaheng  \nPublication Date  \n2024  \nCopyright Information  \nThis work is made available under the terms of a Creative Commons AttributionNonCommercial License, available at [https://creativecommons.org/licenses/by-nc/4.0/](https://creativecommons.org/licenses/by-nc/4.0/)  \n[Peer reviewed|Thesis/dissertation](Peer reviewed|Thesis/dissertation)  \n[eScholarship.org](eScholarship.org) Powered by the California Digital Library  \nUniversity of California  \nUNIVERSITY OF CALIFORNIA  \nSANTA CRUZ  \nTRUSTWORTHY MACHINE LEARNING UNDER IMPERFECT DATA: DATA COLLECTION, ROBUST LEARNING, AND REAL-WORLD CONSIDERATIONS  \nA dissertation submitted in partial satisfaction of the requirements for the degree of  \nDOCTOR OF PHILOSOPHY  \nin  \nCOMPUTER SCIENCE AND ENGINEERING  \nby  \nJiaheng Wei  \nSeptember 2024  \nThe Dissertation of Jiaheng Wei is approved:  \nSenior Research Scientist Gang Niu  \nPeter Biehl  \nVice Provost and Dean of Graduate Studies  \nCopyright © by Jiaheng Wei  \n2024  \nTable of Contents  \nList of Figures vii  \nList of Tables x  \nAbstract xv  \nAcknowledgments xviii  \n1 Introduction 1  \n1.1 Related Works ................................ 5  \n1.2 Preliminary .................................. 11  \n1.2.1 Class-Dependent Label Noise .................... 12  \n1.2.2 Maximizing f−divergence ...................... 13  \n1.2.3 Learning with Smoothed Labels ................... 14  \n1.2.4 Model Confidence ........................... 15  \n1.3 Outline and Completed Works ........................ 15  \n2 Trustworthy Evaluation Mechanisms in Data Collection 18  \n2.1 Credible Sample Elicitation ......................... 18  \n2.1.1 Challenges in Sample Elicitation and Motivation ......... 19  \n2.1.2 Multi-Sample Elicitation with Ground-Truth Samples ...... 20  \n2.1.3 Sample Elicitation with only Peer Samples ............. 22  \n2.1.4 Experiment results .......................... 23  \n2.2 Application to Text Data Evaluation (LLM Hallucination) ........ 27  \n2.2.1 LLM Hallucination .......................... 27  \n2.2.2 Factualness Evaluations via Weighting LLMs ........... 28  \n2.2.3 Experiemnt Results of LLM Hallucination Evaluation ...... 37  \n3 Robust Learning under Label Noise 44  \n3.1 Robust f−divergence ............................. 44  \n3.1.1 Learning with Noisy Labels Using f-divergence .......... 45  \n3.1.2 Variational Difference with Noisy Labels .............. 49  \niii  \n3.1.3 When Df is Robust with Label Noise ................ 53  \n3.1.4 Empirical Results of f−divergence Measures ............ 58  \n3.2 Robust Generalized Label Smoothing .................... 65  \n3.2.1 Learning with GLS .......................... 66  \n3.2.2 Connection to Other Robust Methods ............... 67  \n3.2.3 GLS and Model Confidence ..................... 69  \n3.2.4 GLS with Noisy Labels ....................... 72  \n3.2.5 Experiment Results of GLS ..................... 79  \n4 Real-World Considerations: A Study Using Real-World Human Annotations 90  \n4.1 Human Annotated Noisy Labels on CIFAR-10, CIFAR-100 ........ 91  \n4.1.1 CIFAR-10N Real-World Noisy Label Benchmark ......... 91  \n4.1.2 CIFAR-100N Real-World Noisy Label Benchmark ......... 92  \n4.2 Preliminary Observations on CIFAR-10N, CIFAR-100N ......... 94  \n4.3 Human Noisy Labels v.s. Synthetic Noisy Labels ............. 97  \n4.3.1 A Qualitative Aspect ......................... 98  \n4.3.2 A Quantitative Aspect ........................ 100  \n4.4 Learning with CIFAR-10N and CIFAR-100N ................ 103  \n4.4.1 Performance Comparisons on CIFAR-10N and CIFAR-100N ... 103  \n5 Robust Solutions to Additional Real-World Challenges 108  \n5.1 Challenge 1: Leaning with Multiple Noisy Labels ............. 108  \n5.1.1 Formu","cbCaikQhfJeuULLX","https://ap.wps.com/l/cbCaikQhfJeuULLX","pdf",11006820,1,337,"English","en",105,"# Introduction\n## Related Works\n## Preliminary\n## Outline and Completed Works\n# Trustworthy Evaluation Mechanisms in Data Collection\n## Credible Sample Elicitation\n## Application to Text Data Evaluation (LLM Hallucination)\n# Robust Learning under Label Noise\n## Robust f-divergence\n## Robust Generalized Label Smoothing\n# Real-World Considerations: A Study Using Real-World Human Annotations\n## Human Annotated Noisy Labels on CIFAR-10, CIFAR-100\n## Learning with CIFAR-10N and CIFAR-100N\n# Robust Solutions to Additional Real-World Challenges\n## Multiple Noisy Labels\n## Prior Shifts in Data Distribution\n# Conclusion and Future Works","[{\"question\":\"What problem does this dissertation address in trustworthy machine learning?\",\"answer\":\"It addresses how to build reliable machine learning evaluation and training when data are imperfect, especially under noisy labels and biased data collection.\"},{\"question\":\"How are credible evaluation mechanisms designed for data collection?\",\"answer\":\"The dissertation proposes credible sample elicitation approaches, including multi-sample elicitation with ground-truth samples and strategies using only peer samples.\"},{\"question\":\"Which real-world datasets and challenges are used to validate the methods?\",\"answer\":\"It studies real-world human annotations using CIFAR-10N and CIFAR-100N, and also examines additional challenges including multiple noisy labels and prior shifts in data distribution.\"}]","Trustworthy Machine Learning under Imperfect Data - Data Collection, Robust Learning, and Real-World Considerations | PDF",1785895637,849,{"code":4,"msg":31,"data":32},"ok",{"site_id":24,"language":23,"slug":33,"title":13,"keywords":34,"description":14,"schema_data":35,"social_meta":86,"head_meta":88,"extra_data":90,"updated_unix":28},"trustworthy-machine-learning-under-imperfect-data-data-collection-robust-learning-and-real-world-considerations","",{"@graph":36,"@context":85},[37,54,68],{"@type":38,"itemListElement":39},"BreadcrumbList",[40,44,48,51],{"item":41,"name":42,"@type":43,"position":20},"https://docshare.wps.com","Home","ListItem",{"item":45,"name":46,"@type":43,"position":47},"https://docshare.wps.com/document/","Document",2,{"item":49,"name":12,"@type":43,"position":50},"https://docshare.wps.com/document/research-report/",3,{"item":52,"name":13,"@type":43,"position":53},"https://docshare.wps.com/document/trustworthy-machine-learning-under-imperfect-data-data-collection-robust-learning-and-real-world-considerations/124961/",4,{"url":52,"name":13,"@type":55,"author":56,"headline":13,"publisher":58,"fileFormat":61,"inLanguage":23,"description":14,"dateModified":62,"datePublished":62,"encodingFormat":61,"isAccessibleForFree":63,"interactionStatistic":64},"DigitalDocument",{"name":9,"@type":57},"Person",{"url":41,"name":59,"@type":60},"DocShare","Organization","application/pdf","2026-08-05",true,{"@type":65,"interactionType":66,"userInteractionCount":4},"InteractionCounter",{"@type":67},"ViewAction",{"@type":69,"mainEntity":70},"FAQPage",[71,77,81],{"name":72,"@type":73,"acceptedAnswer":74},"What problem does this dissertation address in trustworthy machine learning?","Question",{"text":75,"@type":76},"It addresses how to build reliable machine learning evaluation and training when data are imperfect, especially under noisy labels and biased data collection.","Answer",{"name":78,"@type":73,"acceptedAnswer":79},"How are credible evaluation mechanisms designed for data collection?",{"text":80,"@type":76},"The dissertation proposes credible sample elicitation approaches, including multi-sample elicitation with ground-truth samples and strategies using only peer samples.",{"name":82,"@type":73,"acceptedAnswer":83},"Which real-world datasets and challenges are used to validate the methods?",{"text":84,"@type":76},"It studies real-world human annotations using CIFAR-10N and CIFAR-100N, and also examines additional challenges including multiple noisy labels and prior shifts in data distribution.","https://schema.org",{"og:url":52,"og:type":87,"og:title":13,"og:site_name":59,"og:description":14},"article",{"robots":89,"canonical":52},"index,follow",{"doc_id":7,"site_id":24},{"code":4,"msg":5,"data":92},[93,97,101,105,110,115,120,123,128,131,135],{"id":20,"doc_module":4,"doc_module_name":46,"category_name":94,"show_sort_weight":95,"slug":96},"Story & Novel",90,"story-novel",{"id":47,"doc_module":4,"doc_module_name":46,"category_name":98,"show_sort_weight":99,"slug":100},"Literature",80,"literature",{"id":53,"doc_module":4,"doc_module_name":46,"category_name":102,"show_sort_weight":103,"slug":104},"Exam",70,"exam",{"id":106,"doc_module":4,"doc_module_name":46,"category_name":107,"show_sort_weight":108,"slug":109},5,"Comic",60,"comic",{"id":111,"doc_module":4,"doc_module_name":46,"category_name":112,"show_sort_weight":113,"slug":114},6,"Technology",50,"technology",{"id":116,"doc_module":4,"doc_module_name":46,"category_name":117,"show_sort_weight":118,"slug":119},7,"Healthcare",40,"healthcare",{"id":11,"doc_module":4,"doc_module_name":46,"category_name":12,"show_sort_weight":121,"slug":122},30,"research-report",{"id":124,"doc_module":4,"doc_module_name":46,"category_name":125,"show_sort_weight":126,"slug":127},9,"Religion & Spirituality",20,"religion-spirituality",{"id":126,"doc_module":4,"doc_module_name":46,"category_name":129,"show_sort_weight":126,"slug":130},"World Cup","world-cup",{"id":132,"doc_module":4,"doc_module_name":46,"category_name":133,"show_sort_weight":132,"slug":134},10,"Lifestyle","lifestyle",{"id":136,"doc_module":4,"doc_module_name":46,"category_name":137,"show_sort_weight":106,"slug":138},19,"General","general"]