[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-327361-105":59,"doc-detail-327361-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","pre-flight-a-benchmark-for-evaluating-large-language-models-on-aviation-operational-knowledge","Pre-Flight: A Benchmark for Evaluating Large Language Models on Aviation Operational Knowledge","","Large language models (LLMs) are increasingly considered for aviation business operations, including documentation, training content generation, and customer-facing assistants. General-purpose benchmarks fail to test whether models reason safely and correctly about aviation-specific operational knowledge, which is critical in a regulated, high-stakes domain. Pre-Flight is an open benchmark with 300 multiple-choice questions sourced from international standards and airport ground operations materials, covering regulations, general aviation knowledge, and complex scenarios. Results show a persistent reliability gap versus an expert reference and provide published datasets and an evaluation harness to support responsible deployment.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/pre-flight-a-benchmark-for-evaluating-large-language-models-on-aviation-operational-knowledge/327361/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/pre-flight-a-benchmark-for-evaluating-large-language-models-on-aviation-operational-knowledge/327361.png","ImageObject",300,407,{"name":92,"@type":93},"Aurora","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-12","2026-09-21",true,{"@type":102,"interactionType":103,"userInteractionCount":52},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What problem does the Pre-Flight benchmark address in aviation LLM evaluation?","Question",{"text":112,"@type":113},"General-purpose benchmarks do not verify whether a model reasons safely and correctly about aviation-specific operational knowledge, leaving a consequential gap for a regulated, high-stakes domain.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"How is the Pre-Flight dataset constructed and what does it cover?",{"text":117,"@type":113},"Pre-Flight provides 300 multiple-choice questions drawn from international standards and airport ground operations materials, spanning international airport ground operations, ICAO and US FAA regulations, aviation general knowledge, and complex operational scenarios authored and reviewed by practitioners.",{"name":119,"@type":110,"acceptedAnswer":120},"What evaluation method and scoring approach does the paper use?",{"text":121,"@type":113},"The paper evaluates contemporary commercial and open-weight models using the Inspect evaluation framework, scoring accuracy under a standard multiple-choice protocol and maintaining a rolling leaderboard as new models are released.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},327361,1791167162,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":52,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":44,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":143,"read_time":144,"preview_page_count":4},4810365810221,"https://ap-avatar.wpscdn.com/davatar_155a257f0dc6eb9ab79c44ca47cae57d","arXiv :2607 .0 1829v 1 [ cs .AI] 2 Jul 2026  \nPre-Flight: A Benchmark for Evaluating Large Language Models on  \nAviation Operational Knowledge  \nAlex Brooker (Airside Labs, London, United Kingdom) Tim Hughes (Mahino Research, Christchurch, New Zealand)  \n29 June 2026 · Licensed under CC BY 4 .0  \nAbstract  \nLarge language models (LLMs) are increasingly proposed for aviation business operations, from documentation and training generation to customer facing assistants. General purpose benchmarks do not measure whether a model reasons safely and correctly about aviation specific operational knowledge, and the high stakes, regulated nature of the domain makes that gap consequential. We present Pre-Flight, an open source benchmark of 300 multiple choice questions drawn from international standards and airport ground operations material, covering international airport ground operations, ICAO and US FAA regulations, aviation general knowledge and complex operational scenarios. Questions were authored and reviewed by practitioners with experience in air traffic management, ground operations and commercial flying. We evaluate a range of contemporary commercial and open weight models using the Inspect evaluation framework, scoring by accuracy under a standard multiple choice protocol, and we maintain the leaderboard on a rolling basis as new models are released. Against an informal expert reference of around 95%, obtained from a low sample quiz of aviation professionals at a conference, even the strongest model evaluated (released in 2026) reaches 82.7%, having improved only gradually from roughly 75% in early 2025 . A substantial and persistent gap below expert level reliability therefore remains. We release the dataset, the evaluation harness and the results, and the benchmark is available within the community evaluations package distributed with inspect_evals. We argue that domain specific evaluation of this kind is a necessary precondition for responsible deployment of generative AI in non safety critical aviation operations.  \nHeadline figures are taken from the leaderboard snapshot of 29 June 2026 (Table 2) .  \n1. Introduction  \nAviation business operations span several subdomains, each with distinct and demanding characteristics. Network planning processes large volumes of demographic and market data to optimise routes and schedules. Airport operations require continuous, forward looking resource balancing in real time. Asset management ranges from short term maintenance tracking to multidecade fleet lifecycle analysis. Airspace and procedure design integrates many disparate data formats with operational expertise under complex regulatory constraints.  \nWhat makes these subdomains particularly challenging is their interconnected nature and the heterogeneity of their data. Information encoded in legacy formats such as NOTAMs and Type B messages over AFTN is used alongside modern digital services. Maintenance records persist as scanned paper with wet signaturesand stamps. Considerable effort over the past two decades has gone into rationalising aviation information models, notably System Wide Information Management (SWIM) under ICAO’s Global Air Navigation Plan, but industry uptake has been slow and migration will take decades. The result is a landscape in which simple message structures from the 1960s coexist with heavyweight geospatial XML schemas, and in which a large share of operationally relevant knowledge, including regulations, training material and procedures, remains unstructured text.  \nThis combination of structured and unstructured information is precisely where generative AI appears promising. The motivating question is whether LLMs can bring these sources together to answer operational questions that were previously intractable, and, critically, whether we can trust the answers. Many aviation decisions are high stakes, whether because of the financial sums involved or the need for accuracy, and errors can carry reput","cbCaig36GCxkCkNZ","https://ap.wps.com/l/cbCaig36GCxkCkNZ","pdf",339429,"English","# Introduction\n## Background and related work\n### What a good LLM evaluation requires","[{\"question\":\"What problem does the Pre-Flight benchmark address in aviation LLM evaluation?\",\"answer\":\"General-purpose benchmarks do not verify whether a model reasons safely and correctly about aviation-specific operational knowledge, leaving a consequential gap for a regulated, high-stakes domain.\"},{\"question\":\"How is the Pre-Flight dataset constructed and what does it cover?\",\"answer\":\"Pre-Flight provides 300 multiple-choice questions drawn from international standards and airport ground operations materials, spanning international airport ground operations, ICAO and US FAA regulations, aviation general knowledge, and complex operational scenarios authored and reviewed by practitioners.\"},{\"question\":\"What evaluation method and scoring approach does the paper use?\",\"answer\":\"The paper evaluates contemporary commercial and open-weight models using the Inspect evaluation framework, scoring accuracy under a standard multiple-choice protocol and maintaining a rolling leaderboard as new models are released.\"}]","Pre-Flight: A Benchmark for Evaluating Large Language Models on Aviation Operational Knowledge | PDF",1789977063,23]