[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-296276-105":59,"doc-detail-296276-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","the-qiyas-benchmark-measuring-chatgpt-mathematical-and-language-understanding-in-arabic-abstract","The Qiyas Benchmark - Measuring ChatGPT Mathematical and Language Understanding in Arabic - Abstract","","Despite the growing importance of Arabic, few language models are pre-trained exclusively on Arabic data, leaving limited benchmarks for Arabic evaluation. This work introduces two new Arabic-native benchmarks, derived from the Qiyas General Aptitude Test, to measure mathematical reasoning and Arabic language understanding. ChatGPT-3.5-turbo and ChatGPT-4 are evaluated on these benchmarks, yielding 49% and 64% average accuracy respectively, revealing substantial difficulty and motivating improved low-resource Arabic model training.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/the-qiyas-benchmark-measuring-chatgpt-mathematical-and-language-understanding-in-arabic-abstract/296276/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/the-qiyas-benchmark-measuring-chatgpt-mathematical-and-language-understanding-in-arabic-abstract/296276.png","ImageObject",300,407,{"name":92,"@type":93},"Caleb Sterling","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-28","2026-09-18",true,{"@type":102,"interactionType":103,"userInteractionCount":44},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"Why are Arabic-native benchmarks needed for evaluating language models?","Question",{"text":112,"@type":113},"Because there is a shortage of large language models pre-trained exclusively on Arabic data, resulting in limited and less robust benchmarks for assessing Arabic performance.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What are the two components of the Qiyas benchmark?",{"text":117,"@type":113},"Qiyas includes a quantitative section for math skills and a verbal section for evaluating Arabic language understanding.",{"name":119,"@type":110,"acceptedAnswer":120},"How did ChatGPT-3.5-turbo and ChatGPT-4 perform on the Qiyas benchmark?",{"text":121,"@type":113},"ChatGPT-4 achieved an overall average accuracy of 64%, while ChatGPT-3.5-turbo achieved an overall accuracy of 49% across different question types.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},296276,1790615633,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":44,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":44,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":143,"read_time":144,"preview_page_count":4},962084925290,"https://ap-avatar.wpscdn.com/davatar_085a072bc5b1113ac321206ff7593b45","The Qiyas Benchmark: Measuring ChatGPT Mathematical and Language Understanding in Arabic  \nShahad Al-Khalifa  \niWAN Research Group King Saud University Riyadh, Saudi Arabia [shahadalkhalifa90@gmail.com](shahadalkhalifa90@gmail.com)  \nHend Al-Khalifa  \nDepartment of Information Technology King Saud University Riyadh, Saudi Arabia [hendk@ksu.edu.sa](hendk@ksu.edu.sa)  \nAbstract  \nDespite the growing importance of Arabic as a global language, there is a notable lack of language models pre-trained exclusively on Arabic data. This shortage has led to limited benchmarks available for assessing language model performance in Arabic. To address this gap, we introduce two novel benchmarks designed to evaluate models'mathematical reasoning and language understanding abilities in Arabic. These benchmarks are derived from a General Aptitude Test (GAT) called Qiyas exam, a standardized test widely used for university admissions in Saudi Arabia. For validation purposes, we assess the performance of ChatGPT-3.5-turbo and ChatGPT-4 on our benchmarks. Our findings reveal that these benchmarks pose a significant challenge, with ChatGPT-4 achieving an overall average accuracy of 64%, while ChatGPT- 3.5-turbo achieved an overall accuracy of 49% across the various question types in the Qiyas benchmark. We believe the release of these benchmarks will pave the way for enhancing the mathematical reasoning and language understanding capabilities of future models tailored for the low-resource Arabic language.  \n1 Introduction  \nEvaluating the capabilities of large language models (LLMs) across different tasks like mathematical reasoning and natural language understanding is a critical area of research as these general-purpose AI systems become more widely used. Developing comprehensive evaluation benchmarks, especially for languages beyond English, is crucial for driving model improvement.  \nArabic, a Semitic language with complex morphology and written from right-to-left, is spoken by over 400 million people across the  \nArab world (Kaye, 2018) . Despite its status as a global language of importance, Arabic is considered low-resource in the field of natural language processing (Almansor et al., 2020) . There is currently a shortage of LLMs pre-trained exclusively on large Arabic datasets. This has resulted in limited benchmarks available for robustly assessing Arabic LLM performance. While some prior work has translated English benchmarks, there is a need for high-quality, natively developed Arabic evaluation resources.  \nThe lack of robust, Arabic-native benchmarks focused on key capabilities like math reasoning and language understanding represents a significant gap in the field. Having professionally designed evaluation resources in this domain could accelerate the development of higherperforming Arabic language models. This work makes two key contributions to address this need:  \n1) It introduces Qiyas, a benchmark suite for comprehensively evaluating LLM performance on mathematical and language tasks in the Arabic language. Qiyas consists of two components - a quantitative section assessing math skills and a verbal section evaluating Arabic language understanding abilities.  \n2) Using the Qiyas benchmark, the performance of the latest ChatGPT models (versions 3.5-turbo and 4) is extensively evaluated across zero-shot, one-shot, and few-shot settings to establish strong Arabic language baselines. These benchmarks are derived from the Qiyas  \nexam, a standardized exam widely used for university admissions in Saudi Arabia, ensuring their quality has been validated by educational experts. The results shed light on current LLM limitations for the Arabic language and highlight the impact of varied training data and prompting approaches. The release of Qiyas paves the way for advancing Arabic LLMs' mathematical reasoning and language understanding  \ncapabilities on these challenging, nationally representative Arabic tasks.  \nThe remainder of the paper is","cbCairswgKaDJxxb","https://ap.wps.com/l/cbCairswgKaDJxxb","pdf",538228,"English","# Introduction\n# Background and Literature Review\n# Dataset Description\n# Evaluation Approach\n# Results and Discussion\n# Error Analysis\n# Conclusion","[{\"question\":\"Why are Arabic-native benchmarks needed for evaluating language models?\",\"answer\":\"Because there is a shortage of large language models pre-trained exclusively on Arabic data, resulting in limited and less robust benchmarks for assessing Arabic performance.\"},{\"question\":\"What are the two components of the Qiyas benchmark?\",\"answer\":\"Qiyas includes a quantitative section for math skills and a verbal section for evaluating Arabic language understanding.\"},{\"question\":\"How did ChatGPT-3.5-turbo and ChatGPT-4 perform on the Qiyas benchmark?\",\"answer\":\"ChatGPT-4 achieved an overall average accuracy of 64%, while ChatGPT-3.5-turbo achieved an overall accuracy of 49% across different question types.\"}]","The Qiyas Benchmark - Measuring ChatGPT Mathematical and Language Understanding in Arabic - Abstract | PDF",1789730459,23]