[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-156011-105":59,"doc-detail-156011-en":131},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":124,"head_meta":126,"extra_data":128,"updated_unix":130},105,"en","assessing-sparql-capabilities-of-large-language-models","Assessing SPARQL Capabilities of Large Language Models","","Integration of Large Language Models (LLMs) with Knowledge Graphs (KGs) enables knowledge-driven applications, especially by interpreting and generating formal Semantic Web languages where SPARQL is central for accessing RDF-based graphs. This paper evaluates out-of-the-box LLM performance on SPARQL SELECT queries using quantitative benchmarking. Benchmarks are implemented in the LLM-KG-Bench framework, testing syntax, semantic read, semantic create, and the effect of knowledge-graph prompt inclusion. Results show SPARQL SELECT remains challenging and depends strongly on task complexity and model choice.",{"@graph":69,"@context":123},[70,84,106],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/assessing-sparql-capabilities-of-large-language-models/156011/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/assessing-sparql-capabilities-of-large-language-models/156011.png","ImageObject",300,407,{"name":92,"@type":93},"Theodora","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-27","2026-08-28",true,{"@type":102,"interactionType":103,"userInteractionCount":105},"InteractionCounter",{"@type":104},"ViewAction",16,{"@type":107,"mainEntity":108},"FAQPage",[109,115,119],{"name":110,"@type":111,"acceptedAnswer":112},"What does the paper measure about LLMs in relation to SPARQL?","Question",{"text":113,"@type":114},"It quantitatively benchmarks out-of-the-box LLM capabilities on SPARQL SELECT queries, focusing on syntax, semantic read, semantic create, and how knowledge-graph prompt inclusion affects results.","Answer",{"name":116,"@type":111,"acceptedAnswer":117},"Which framework is used to run and evaluate the benchmarking tasks?",{"text":118,"@type":114},"The tasks are implemented and executed in the LLM-KG-Bench framework for automated execution and evaluation across several LLMs.",{"name":120,"@type":111,"acceptedAnswer":121},"What main conclusion does the paper report for SPARQL SELECT query generation?",{"text":122,"@type":114},"Working with SPARQL SELECT queries is still challenging for LLMs, and performance varies heavily with both the specific model and the complexity of the task.","https://schema.org",{"og:url":83,"og:type":125,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":127,"canonical":83},"index,follow",{"doc_id":129,"site_id":62},156011,1787942672,{"code":4,"msg":5,"data":132},{"doc_id":129,"user_id":133,"nickname":92,"user_avatar":134,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":105,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":56,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":130,"read_time":144,"preview_page_count":4},687197207919,"https://ap-avatar.wpscdn.com/avatar/a000253d6f5f7c60be?x-image-process=image/resize,m_fixed,w_180,h_180&k=1779446848396160552","Assessing SPARQL Capabilities of Large Language Models  \nLars-Peter Meyer1,2,∗ , Johannes Frey1,2 , Felix Brei1 and Natanael Arndt1  \n1 Institute for Applied Informatics at Leipzig University, Goerdelerring 9, 04109 Leipzig, Germany  \n2 Leipzig University, Germany  \nAbstract  \nThe integration of Large Language Models (LLMs) with Knowledge Graphs (KGs) offers significant synergistic potential for knowledge-driven applications. One possible integration is the interpretation and generation of formal languages, such as those used in the Semantic Web, with SPARQL being a core technology for accessing KGs. In this paper, we focus on measuring out-of-the box capabilities of LLMs to work with SPARQL and more specifically with SPARQL SELECT queries applying a quantitative approach.  \nWe implemented various benchmarking tasks in the LLM-KG-Bench framework for automated execution and evaluation with several LLMs. The tasks assess capabilities along the dimensions of syntax, semantic read, semantic create, and the role of knowledge graph prompt inclusion.  \nWith this new benchmarking tasks, we evaluated a selection of GPT, Gemini, and Claude models. Our findings indicate that working with SPARQL SELECT queries is still challenging for LLMs and heavily depends on the specific LLM as well as the complexity of the task. While fixing basic syntax errors seems to pose no problems for the best of the current LLMs evaluated, creating semantically correct SPARQL SELECT queries is difficult in several cases.  \nKeywords  \nLLM benchmarking, SPARQL, LLMs for Knowledge Graphs, RDF, LLM, Knowledge Graph  \n1. Introduction  \nThe combination of Large Language Models (LLMs) and Knowledge Graphs (KGs) is still gaining more traction as the rapidly growing number of research articles 1 and the inclusion of LLMs in several KG related conference calls show. This includes different combinations [1] of LLMs and KGs, especially LLMs supported by KGs and LLMs supporting the work with KGs. An important interface for both combinations is the SPARQL query language as the structured retrieval and update interface for RDF KGs [2] .  \nWith the fast evolving list of LLMs available, the need for automated benchmarks increases. LLM performances is evaluated by BigBench [3], on the Open LLM Leaderboard [4], and the Chatbot-Arena [5], but these are very general and do not focus on KGs or SPARQL. In existing  \nNLP4KGC: 3rd International Workshop on Natural Language Processing for Knowledge Graph Creation, September 17, 2024, Amsterdam, Netherlands  \n∗Corresponding author.  \n$ [lpmeyer@infai.org](lpmeyer@infai.org) (L. Meyer); [frey@informatik.uni-leipzig.de](frey@informatik.uni-leipzig.de) (J. Frey); [brei@infai.org](brei@infai.org) (F. Brei)  \n􀀚 0000-0001-5260-5181 (L. Meyer); 0000-0003-3127-0815 (J. Frey); 0009-0008-5245-6655 (F. Brei);  \n0000-0002-8130-8677 (N. Arndt)  \n © 2024 Copyright for this paper by its authors. Use permitted under Creative Commons License Attribution 4.0 International (CC BY 4.0) .  \n1 list of articles combining KG+LLM: [https://github.com/zjukg/KG-LLM-Papers](https://github.com/zjukg/KG-LLM-Papers)  \n1  \nCEUR ~~  ~~[Workshop](Workshop ceur-ws.org)[ ceur-ws.org](Workshop ceur-ws.org)[ ](Workshop ceur-ws.org)[Proceedings](Proceedings ISSN 1613-0073)[ ISSN 1613-0073](Proceedings ISSN 1613-0073)   \nwrite  \nread  \n\n| RQ3-semantic create |\n| --- |\n|  |\n| RQ2-semantic read |\n|  |\n| RQ1-syntax |\n\nFigure 1: An overview on the relation of the research questions.  \nwork, KG related benchmarking is done specifically on the Turtle format [6] or often as discussion and comparison of very specific solutions. LLM benchmarking related to SPARQL was done in a small scale by Meyer et al. [7] and by Kovriguina et al. [8] .  \nIn this paper, a set of automated benchmarking tasks is presented, to assess the basic capabilities of LLMs to deal with SPARQL SELECT queries without special architectures or fine tuning. We reuse existing work on Knowledge Graph Question Answerin","cbCaitGvrtrkDGOs","https://ap.wps.com/l/cbCaitGvrtrkDGOs","pdf",2516526,"English","# Introduction\n## Research Questions\n## Contribution\n# Benchmarking Framework and Task Types\n## SPARQL Syntax Fixing (SSF)\n## Text to SPARQL (T2S)\n## SPARQL to Answer (S2A)\n## Text to Answer (T2A)","[{\"question\":\"What does the paper measure about LLMs in relation to SPARQL?\",\"answer\":\"It quantitatively benchmarks out-of-the-box LLM capabilities on SPARQL SELECT queries, focusing on syntax, semantic read, semantic create, and how knowledge-graph prompt inclusion affects results.\"},{\"question\":\"Which framework is used to run and evaluate the benchmarking tasks?\",\"answer\":\"The tasks are implemented and executed in the LLM-KG-Bench framework for automated execution and evaluation across several LLMs.\"},{\"question\":\"What main conclusion does the paper report for SPARQL SELECT query generation?\",\"answer\":\"Working with SPARQL SELECT queries is still challenging for LLMs, and performance varies heavily with both the specific model and the complexity of the task.\"}]","Assessing SPARQL Capabilities of Large Language Models | PDF",48]