[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-178784-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-178784-en":131},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","2024-findings-naacl-149","2024 Findings - NAACL 149","","Document analyzes model performance on standardized exams and their sub-tasks across languages (Chinese and English). It reports aggregate statistics such as number of participants, token averages, and accuracy under multiple prompting regimes including zero-shot, zero-shot CoT, few-shot, few-shot CoT, plus separate prompting templates. Coverage spans geography, biology, history, chemistry, physics, English, math, logic, law analytics/logic/reading, LSAT sections, SAT math/English, and Gaokao and SAT benchmarks, presenting comparative results by task and setting.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/2024-findings-naacl-149/178784/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/2024-findings-naacl-149/178784.png","ImageObject",300,407,{"name":42,"@type":43},"Genevieve","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-10-27","2026-09-02",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",12,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"这份文档主要评估什么？","Question",{"text":63,"@type":64},"围绕多类标准化考试及其子任务评估模型表现，包括数学、逻辑、法学、以及中英文科目相关题型。","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"文档中使用了哪些提示策略？",{"text":68,"@type":64},"包含零样本提示（Zero-shot）、零样本思维链（Zero-shot CoT）、少样本提示（Few-shot）与少样本思维链（Few-shot CoT），并展示对应的提示输入模板。",{"name":70,"@type":61,"acceptedAnswer":71},"结果通常以哪些指标呈现？",{"text":72,"@type":64},"按任务汇总给出正确率/Top表现等统计，并在不同提示方式与模型设置下进行对比，同时也记录参与者规模与平均token等信息。","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},178784,1788333475,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,111,115,120,123,127],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":108,"show_sort_weight":109,"slug":110},7,"Healthcare",40,"healthcare",{"id":112,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":113,"slug":114},8,30,"research-report",{"id":116,"doc_module":4,"doc_module_name":25,"category_name":117,"show_sort_weight":118,"slug":119},9,"Religion & Spirituality",20,"religion-spirituality",{"id":118,"doc_module":4,"doc_module_name":25,"category_name":121,"show_sort_weight":118,"slug":122},"World Cup","world-cup",{"id":124,"doc_module":4,"doc_module_name":25,"category_name":125,"show_sort_weight":124,"slug":126},10,"Lifestyle","lifestyle",{"id":128,"doc_module":4,"doc_module_name":25,"category_name":129,"show_sort_weight":97,"slug":130},19,"General","general",{"code":4,"msg":82,"data":132},{"doc_id":79,"user_id":133,"nickname":42,"user_avatar":134,"doc_module":4,"category_id":112,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":140,"language":141,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":12,"update_tm":80,"read_time":109,"preview_page_count":4},1374391974585,"https://ap-avatar.wpscdn.com/davatar_276721f389ce27ea32af1340a28f341c","| Exams | \\#Participants | Language | Tasks | Subject | \\# Instance | \\#Avg. Token |\n| --- | --- | --- | --- | --- | --- | --- |\n| Gaokao | 12M | Chinese | GK-geography\u003Cbr>GK-biology\u003Cbr>GK-history\u003Cbr>GK-chemistry\u003Cbr>GK-physics\u003Cbr>GK-En\u003Cbr>GK-Ch\u003Cbr>GK-Math-QA\u003Cbr>GK-Math-Cloze | Geography Biology History Chemistry Physics English Chinese Math\u003Cbr>Math | 199\u003Cbr>210\u003Cbr>243\u003Cbr>207\u003Cbr>200\u003Cbr>306\u003Cbr>246\u003Cbr>351\u003Cbr>118 | 144\u003Cbr>141\u003Cbr>116\u003Cbr>113\u003Cbr>124\u003Cbr>356\u003Cbr>935\u003Cbr>68\u003Cbr>60 |\n| SAT | 1.7M | English | SAT-En.\u003Cbr>SAT-Math | English Math | 206\u003Cbr>220 | 656\u003Cbr>54 |\n| Lawyer Qualification Test | 820K | Chinese | JEC-QA-KD\u003Cbr>JEC-QA-CA | Law\u003Cbr>Law | 1000\u003Cbr>1000 | 146\u003Cbr>213 |\n| Law School\u003Cbr>Admission Test (LSAT) | 170K | English | LSAT-AR\u003Cbr>LSAT-LR\u003Cbr>LSAT-RC | Law-Analytics\u003Cbr>Law-Logic\u003Cbr>Law-Reading | 230\u003Cbr>510\u003Cbr>260 | 154\u003Cbr>178\u003Cbr>581 |\n| Civil Service Examination | 2M\u003Cbr>2M | English\u003Cbr>Chinese | LogiQA-en\u003Cbr>LogiQA-ch | Logic\u003Cbr>Logic | 651\u003Cbr>651 | 144\u003Cbr>242 |\n| GREGMAT | 340K\u003Cbr>150K | English\u003Cbr>English | AQuA-RAT | Math | 254 | 77 |\n| AMC\u003Cbr>AIME | 300K\u003Cbr>3000 | English\u003Cbr>English | MATH | Math | 1000 | 40 |\n\n\n| Task/Model | Human | Zero-Shot |  |  | Zero-Shot CoT |  |  | Few-Shot |  |  | Few-Shot CoT |  |  |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n|  | Avg. Top | TD | CG | G4 | TD | CG | G4 | TD | CG | G4 | TD | CG | G4 |\n| AQuA-RAT | 85 100 | 29.9 | 31.9 | 40.6 | 42.1 | 55.9 | 73.2 | 30.3 | 31.1 | 50.8 | 47.2 | 60.6 | 74.0 |\n| MATH | 40 90 | 11.9 | 26.4 | 35.7 | 19.1 | 31.9 | 47.7 | 10.3 | 14.8 | 15.1 | 15.1 | 30.1 | 25.3 |\n| LogiQA (English) | 86 95 | 22.7 | 35.0 | 49.3 | 36.9 | 39.9 | 57.8 | 43.5 | 43.5 | 63.9 | 37.5 | 38.9 | 62.7 |\n| LogiQA (Chinese) | 88 96 | 40.3 | 41.0 | 58.8 | 36.7 | 38.9 | 57.5 | 43.2 | 46.2 | 65.0 | 40.0 | 38.6 | 61.9 |\n| JEC-QA-KD | 71 78 | 21.9 | 21.1 | 33.4 | 18.4 | 21.2 | 31.9 | 22.4 | 27.6 | 41.3 | 23.6 | 23.4 | 40.4 |\n| JEC-QA-CA | 58 85 | 21.0 | 22.0 | 31.1 | 16.7 | 19.6 | 29.8 | 22.2 | 25.1 | 37.4 | 16.1 | 20.0 | 34.7 |\n| LSAT-AR | 56 91 | 21.7 | 24.4 | 35.2 | 23.9 | 22.6 | 34.4 | 22.6 | 25.7 | 33.9 | 22.6 | 25.2 | 31.7 |\n| LSAT-LR | 56 91 | 47.5 | 52.6 | 80.6 | 50.0 | 52.6 | 80.6 | 60.4 | 59.2 | 85.9 | 51.2 | 52.2 | 84.5 |\n| LSAT-RC | 56 91 | 64.7 | 65.4 | 85.9 | 57.6 | 62.1 | 85.1 | 70.6 | 67.7 | 87.7 | 64.3 | 57.6 | 87.7 |\n| SAT-Math | 66 94 | 35.5 | 42.7 | 64.6 | 54.6 | 70.9 | 95.0 | 44.6 | 40.9 | 71.8 | 55.5 | 65.0 | 89.6 |\n| SAT-English | 66 94 | 74.8 | 81.1 | 88.8 | 75.7 | 77.7 | 85.9 | 84.0 | 81.1 | 88.8 | 76.7 | 78.2 | 85.9 |\n| GK-Cn | 65 85 | 43.9 | 39.0 | 53.3 | 35.4 | 33.7 | 44.7 | 25.6 | 41.5 | 61.4 | 29.3 | 37.8 | 51.6 |\n| GK-En | 69 91 | 81.4 | 84.9 | 91.9 | 83.0 | 84.3 | 92.5 | 86.9 | 86.3 | 93.8 | 80.7 | 84.6 | 93.1 |\n| GK-geography | 65 85 | 53.3 | 59.8 | 76.9 | 48.7 | 55.8 | 72.4 | 59.8 | 63.8 | 75.9 | 52.3 | 61.8 | 76.4 |\n| GK-history | 64 85 | 47.3 | 59.7 | 77.4 | 37.0 | 50.2 | 76.5 | 49.0 | 57.6 | 77.8 | 51.9 | 58.4 | 78.2 |\n| GK-biology | 68 89 | 40.5 | 52.9 | 75.7 | 30.0 | 42.4 | 71.9 | 44.3 | 52.4 | 80.0 | 32.9 | 50.0 | 72.9 |\n| GK-chemistry | 66 86 | 27.1 | 38.7 | 51.7 | 24.6 | 33.8 | 52.2 | 32.4 | 44.0 | 54.6 | 35.8 | 33.8 | 54.1 |\n| GK-physics | 71 94 | 22.0 | 33.0 | 39.0 | 18.5 | 29.5 | 45.5 | 31.0 | 33.5 | 43.5 | 27.5 | 36.5 | 54.5 |\n| GK-Math-QA | 73 96 | 28.2 | 36.5 | 47.0 | 28.8 | 33.3 | 50.7 | 27.6 | 31.3 | 39.9 | 33.1 | 31.6 | 49.0 |\n| GK-Math-Cloze | 73 96 | 17.0 | 7.6 | 16.1 | 4.2 | 5.1 | 15.3 | 5.9 | 5.9 | 11.0 | 5.93 | 8.5 | 16.1 |\n| Average | 67 91 | 38.1 | 42.9 | 56.4 | 37.4 | 43.2 | 58.4 | 41.2 | 44.4 | 59.2 | 40.4 | 45 | 61.3 |\n\n\n|  |  |  |\n| --- | --- | --- |\n|  | Zero-shot Prompting |  |\n| Input: [Question]\u003Cbr>Among A to D, the answer is:\u003Cbr>Output: \u003CAnswer> |  |  |\n\n\n|  |\n| --- |\n| - Prompting\u003Cbr>Input: Here are the answers for the questions in exams.\u003Cbr>􀀡1 : [Question 1] The answer is [Answer 1]\u003Cbr>􀀡2 : [Question 2] The answer is [Answer 2]\u003Cbr>…\u003Cbr>􀀡􀀤 : [Question 􀀤] The answer is [Answer 􀀤][Qu","cbCaijz7zWdsFOr0","https://ap.wps.com/l/cbCaijz7zWdsFOr0","pdf",409880,16,"English","# Evaluation Setup\n## Exam and Task Coverage\n# Prompting Strategies\n## Zero-shot Prompting\n## Zero-shot CoT\n## Few-shot CoT\n# Results and Analysis\n## Task-level Comparisons","[{\"question\":\"这份文档主要评估什么？\",\"answer\":\"围绕多类标准化考试及其子任务评估模型表现，包括数学、逻辑、法学、以及中英文科目相关题型。\"},{\"question\":\"文档中使用了哪些提示策略？\",\"answer\":\"包含零样本提示（Zero-shot）、零样本思维链（Zero-shot CoT）、少样本提示（Few-shot）与少样本思维链（Few-shot CoT），并展示对应的提示输入模板。\"},{\"question\":\"结果通常以哪些指标呈现？\",\"answer\":\"按任务汇总给出正确率/Top表现等统计，并在不同提示方式与模型设置下进行对比，同时也记录参与者规模与平均token等信息。\"}]","2024 Findings - NAACL 149 | PDF"]