[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-178965-105":59,"doc-detail-178965-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","2023-findings-for-acl-513","2023 Findings for ACL 513","","Research findings from ACL 513 evaluate a set of language models on multiple reasoning and question answering benchmarks, including ReClor, LogiQA, RACE, SQuAD 2.0, STS-B, and MNLI. The study reports development and test performance under different training settings such as PMI&CLI, ATI, and NTI, alongside methods like MLM and LCP-IDOL. Additional experiments compare zero-shot, few-shot, and chain-of-thought prompting templates, with tables summarizing gains and relative model strengths.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/2023-findings-for-acl-513/178965/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/2023-findings-for-acl-513/178965.png","ImageObject",300,407,{"name":92,"@type":93},"Đào","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-06","2026-09-02",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",12,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"Which benchmarks are used to evaluate the language models in ACL 513?","Question",{"text":113,"@type":114},"The document reports results on ReClor, LogiQA, RACE, SQuAD 2.0, STS-B, and MNLI, with development and test splits in several tables.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"How do zero-shot, few-shot, and chain-of-thought settings differ in the experiments?",{"text":118,"@type":114},"The study defines separate prompting templates for each setting: zero-shot uses only the passage and question, few-shot adds multiple examples, and chain-of-thought includes an explicit thought process before selecting an answer.",{"name":120,"@type":111,"acceptedAnswer":121},"What training variants are compared using PMI&CLI, NTI, and ATI?",{"text":122,"@type":114},"The tables compare model performance when trained or evaluated with PMI&CLI versus NTI and ATI components, and they also show the effect of adding methods such as MLM or LCP.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},178965,1788334659,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":140,"language":141,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":142,"faqs":143,"seo_title":144,"seo_description":67,"update_tm":130,"read_time":145,"preview_page_count":4},1374402968488,"https://ap-avatar.wpscdn.com/davatar_29158cc5080c5b710cf443261637dec0","| Type | Library | Example |\n| --- | --- | --- |\n| PMI | given that, seeing that, for the reason that, owing to, as indicated by, on the grounds that, on account of, considering, because of, due to, now that, may be inferred from, by virtue of, in view of, for the sake of, thanks to, as long as, based on that, as a result of, considering that, inasmuch as, if and only if, according to, in that, only if, because, depend on, rely on | The real world contains no political entity exercising literally total control over even one such aspect.  This is because any system of control is inefficient, and, therefore, its degree of control is partial. |\n| CLI | conclude that, entail that, infer that, that is why, therefore, thereby, wherefore, accordingly, hence, thus, consequently, whence, so that, it follows that, imply that, as a result, suggest that, prove that, as a conclusion, conclusively, for this reason, as a consequence, on that account, in conclusion, to that end, because of this, that being so, ergo, in this way, in this manner, by such means, as it turns out, result in, in order that, show that, eventually | In the United States, each bushel of corn produced might  result in  the loss of as much as two bushels of topsoil. Moreover, in the last 100 years, the topsoil in many states, which once was about fourteen inches thick, has been eroded to only six or eight inches. |\n| NTI | not, neither, none of, unable, few, little, hardly, merely, seldom, without, never, nobody, nothing, nowhere, rarely, scarcely, barely, no longer, isn’t, aren’t, wasn’t, weren’t, can’t, cannot, couldn’t, won’t, wouldn’t, don’t, doesn’t, didn’t, haven’t, hasn’t | A high degree of creativity and a high level of artistic skill are  seldom  combined in the creation of a work of art. |\n| ATI | although, though, but, nevertheless, however, instead of, nonetheless, yet, rather, whereas, otherwise, conversely, on the contrary, even, nevertheless, despite, in spite of, in contrast, even if, eventhough, unless, regardless of, reckless of | This advantage accruing to the sentinel does not mean that its watchful behavior is entirely selfinterested.  On the contrary , the sentinel’s behavior is an example of animal behavior motivated at least in part by altruism. |\n| CNI | and, or, nor, also, moreover, in addition, on the other hand, meanwhile, further, afterward, next, besides, additionally, meantime, furthermore, as well, simultaneously, either, both, similarly, likewise | A graduate degree in policymaking is necessary to serve in the presidential cabinet. In addition , everyone in the cabinet must pass a security clearance. |\n\n| Models | ReClor |  | LogiQA |  |\n| --- | --- | --- | --- | --- |\n|  | Dev | Test | Dev | Test |\n| BERT | 53.8 | 49.8 | 35.3♠ | 33.0♠ |\n| IDOL | 56.8 | 53.3 | 36.9 | 34.3 |\n| RoBERTa | 62.6 | 55.6 | 37.0♠ | 36.6♠ |\n| DAGN | 65.2 | 58.2 | 35.5 | 38.7 |\n| AdaLoGN | 65.2 | 60.2 | 39.9 | 40.7 |\n| LReasoner | 66.2 | 62.4 | 38.1 | 40.6 |\n| MERIt | 67.8 | 60.7 | 42.4 | 41.5 |\n| Logiformer | 68.4 | 63.5 | 42.2 | 42.6 |\n| IDOL | 70.2 | 63.9 | 42.5 | 41.8 |\n| ALBERT | 70.4 | 67.3 | 41.2♠ | 41.3♠ |\n| LReasoner | 73.2 | 70.7 | 41.6 | 41.2 |\n| MERIT | 73.2 | 71.1 | 43.9 | 45.3 |\n| IDOL | 74.6 | 70.9 | 44.7 | 43.8 |\n\n\n| Models | Test | ReClor\u003Cbr>Test-E | Test-H |\n| --- | --- | --- | --- |\n| DeBERTa♡ | 75.3 | 84.0 | 68.4 |\n| LReasoner♣ | 76.1 | 87.1 | 67.5 |\n| Knowledge Model♣ | 79.2 | 91.8 | 69.3 |\n| MERIt♣ | 79.3 | 85.2 | 74.6 |\n| AMR-LE♣ | 80.0 | 87.7 | 73.9 |\n| IDOL | 80.6 | 87.7 | 75.0 |\n\n| Models | ReClor |  | LogiQA |  | RACE |  | SQuAD 2.0 |  | STS-B |  | MNLI |  |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n|  | Dev | Test | Dev | Test | Dev Test |  | F1 | EM | Pear. | Spear. | m | mm |\n| RoBERTa | 62.7 | 55.2 | 36.2 | 37.1 | 85.2 | 84.4 | 89.0 | 86.1 | 92.6 | 92.5 | 89.5 | 89.3 |\n| +MLM | 65.0 | 58.4 | 37.9 | 36.6 | 85.4 | 84.5 | 89.0 | 86.1 | 92.2 | 92.1 | 89.5 | 89.5 |\n| +LCP (IDOL) | 66.8 | 60.6 | 39.4 |","cbCaiiWlcZLEFas1","https://ap.wps.com/l/cbCaiiWlcZLEFas1","pdf",970097,13,"English","# Evaluation Benchmarks\n## ReClor and LogiQA\n## RACE, SQuAD 2.0, STS-B, MNLI\n# Training and Prompting Settings\n## PMI&CLI, NTI, ATI Variants\n## Zero-shot, Few-shot, Chain-of-Thought","[{\"question\":\"Which benchmarks are used to evaluate the language models in ACL 513?\",\"answer\":\"The document reports results on ReClor, LogiQA, RACE, SQuAD 2.0, STS-B, and MNLI, with development and test splits in several tables.\"},{\"question\":\"How do zero-shot, few-shot, and chain-of-thought settings differ in the experiments?\",\"answer\":\"The study defines separate prompting templates for each setting: zero-shot uses only the passage and question, few-shot adds multiple examples, and chain-of-thought includes an explicit thought process before selecting an answer.\"},{\"question\":\"What training variants are compared using PMI\\u0026CLI, NTI, and ATI?\",\"answer\":\"The tables compare model performance when trained or evaluated with PMI\\u0026CLI versus NTI and ATI components, and they also show the effect of adding methods such as MLM or LCP.\"}]","2023 Findings for ACL 513 | PDF",33]