[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-seo-142567-105":3,"detail-sidebar-cat-0-en-105":81,"doc-detail-142567-en":131},{"code":4,"msg":5,"data":6},0,"ok",{"site_id":7,"language":8,"slug":9,"title":10,"keywords":11,"description":12,"schema_data":13,"social_meta":74,"head_meta":76,"extra_data":78,"updated_unix":80},105,"en","neurips-2021-proto-program-guided-transformer-for-program-guided-tasks-supplemental","NeurIPS 2021 - Proto Program Guided Transformer for Program Guided Tasks - Supplemental","","Supplemental material analyzing a prototype program-guided transformer for program-guided tasks. Includes visualization of validation and test outcomes on the GQA dataset, highlighting where the MMN baseline fails and the proposed approach succeeds, with attention to reasoning correctness and trace differences. Provides additional related work on masked attention and neural symbolic learning, and describes program details for two experimental tasks, including program-guided visual reasoning and routine structure on GQA.",{"@graph":14,"@context":73},[15,34,56],{"@type":16,"itemListElement":17},"BreadcrumbList",[18,23,27,31],{"item":19,"name":20,"@type":21,"position":22},"https://docshare.wps.com","Home","ListItem",1,{"item":24,"name":25,"@type":21,"position":26},"https://docshare.wps.com/document/","Document",2,{"item":28,"name":29,"@type":21,"position":30},"https://docshare.wps.com/document/research-report/","Research & Report",3,{"item":32,"name":10,"@type":21,"position":33},"https://docshare.wps.com/document/neurips-2021-proto-program-guided-transformer-for-program-guided-tasks-supplemental/142567/",4,{"url":32,"name":10,"@type":35,"image":36,"author":41,"headline":10,"publisher":44,"fileFormat":47,"inLanguage":8,"description":12,"dateModified":48,"datePublished":49,"encodingFormat":47,"isAccessibleForFree":50,"interactionStatistic":51},"DigitalDocument",{"url":37,"@type":38,"width":39,"height":40},"https://docshare.wps.com/thumbnails/neurips-2021-proto-program-guided-transformer-for-program-guided-tasks-supplemental/142567.png","ImageObject",300,407,{"name":42,"@type":43},"Fahsai","Person",{"url":19,"name":45,"@type":46},"DocShare","Organization","application/pdf","2026-10-30","2026-08-25",true,{"@type":52,"interactionType":53,"userInteractionCount":55},"InteractionCounter",{"@type":54},"ViewAction",13,{"@type":57,"mainEntity":58},"FAQPage",[59,65,69],{"name":60,"@type":61,"acceptedAnswer":62},"What do the validation visualizations on the GQA dataset compare?","Question",{"text":63,"@type":64},"They compare reasoning and answer correctness between the MMN approach and the ProTo model across example questions, using green ticks and red crosses to mark correct and wrong reasoning results.","Answer",{"name":66,"@type":61,"acceptedAnswer":67},"How does the ProTo model differ from the MMN baseline on the GQA examples shown?",{"text":68,"@type":64},"The figures indicate cases where ProTo predicts the correct answer while MMN fails, and where reasoning traces can differ even when the final answers match.",{"name":70,"@type":61,"acceptedAnswer":71},"What program details are provided for the experimental tasks?",{"text":72,"@type":64},"The document includes additional details of the programs used on two tasks, including a section on program-guided visual reasoning and the structure of routines in the GQA dataset.","https://schema.org",{"og:url":32,"og:type":75,"og:title":10,"og:site_name":45,"og:description":12},"article",{"robots":77,"canonical":32},"index,follow",{"doc_id":79,"site_id":7},142567,1787683492,{"code":4,"msg":82,"data":83},"success",[84,88,92,96,101,106,111,115,120,123,127],{"id":22,"doc_module":4,"doc_module_name":25,"category_name":85,"show_sort_weight":86,"slug":87},"Story & Novel",90,"story-novel",{"id":26,"doc_module":4,"doc_module_name":25,"category_name":89,"show_sort_weight":90,"slug":91},"Literature",80,"literature",{"id":33,"doc_module":4,"doc_module_name":25,"category_name":93,"show_sort_weight":94,"slug":95},"Exam",70,"exam",{"id":97,"doc_module":4,"doc_module_name":25,"category_name":98,"show_sort_weight":99,"slug":100},5,"Comic",60,"comic",{"id":102,"doc_module":4,"doc_module_name":25,"category_name":103,"show_sort_weight":104,"slug":105},6,"Technology",50,"technology",{"id":107,"doc_module":4,"doc_module_name":25,"category_name":108,"show_sort_weight":109,"slug":110},7,"Healthcare",40,"healthcare",{"id":112,"doc_module":4,"doc_module_name":25,"category_name":29,"show_sort_weight":113,"slug":114},8,30,"research-report",{"id":116,"doc_module":4,"doc_module_name":25,"category_name":117,"show_sort_weight":118,"slug":119},9,"Religion & Spirituality",20,"religion-spirituality",{"id":118,"doc_module":4,"doc_module_name":25,"category_name":121,"show_sort_weight":118,"slug":122},"World Cup","world-cup",{"id":124,"doc_module":4,"doc_module_name":25,"category_name":125,"show_sort_weight":124,"slug":126},10,"Lifestyle","lifestyle",{"id":128,"doc_module":4,"doc_module_name":25,"category_name":129,"show_sort_weight":97,"slug":130},19,"General","general",{"code":4,"msg":82,"data":132},{"doc_id":79,"user_id":133,"nickname":42,"user_avatar":134,"doc_module":4,"category_id":112,"category_name":29,"doc_title":10,"doc_description":12,"doc_content":135,"file_id":136,"file_url":137,"file_type":138,"file_size":139,"view_count":55,"is_deleted":4,"is_public":22,"is_downloadable":22,"audit_status":22,"page_count":118,"language":140,"language_code":8,"site_id":7,"html_lang":8,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":12,"update_tm":80,"read_time":104,"preview_page_count":4},549768702563,"https://ap-avatar.wpscdn.com/avatar/8000c4aa63b76e948b?x-image-process=image/resize,m_fixed,w_180,h_180&k=1786536092046926083","| Image | | |\n| --- | --- | --- |\n| Question | Is the plate blue and round? | Do you see horses near the bushes? |\n| MMN\u003Cbr>Results | Select(plate)\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>Verify(blue) Verify(round)\u003Cbr>\u003Cbr>True False\u003Cbr>\u003Cbr>And()\u003Cbr>\u003Cbr>False ✘ | Select(bush)\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>RelateInverse(near, horse)\u003Cbr>\u003Cbr>\u003Cbr>Exist()\u003Cbr>\u003Cbr>False✘ |\n| ProTo\u003Cbr>Results | Select(plate)\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>Verify(blue) Verify(round)\u003Cbr>\u003Cbr>True True\u003Cbr>\u003Cbr>And()\u003Cbr>\u003Cbr>True ✓ | \u003Cbr>\u003Cbr>\u003Cbr>RelateInverse(near, horse)\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>Exist()\u003Cbr>\u003Cbr>True ✓ |\n\nFigure 1: Visualization of validation results on the GQA dataset (Part 1) . We use the green tick and the red cross to denote right and wrong reasoning result respectively. The MMN approach fails both examples while ProTo successfully addresses them.  \n\n| Image | |  | |\n| --- | --- | --- | --- |\n| Question | Is there a kite or a racket that is white? |  | Is there a young man to the left of a plate? |\n| MMN\u003Cbr>Results | | | Select (plate)\u003Cbr>\u003Cbr>\u003Cbr>Relate(to the right, man)\u003Cbr>\u003Cbr>\u003Cbr>|\n|  | True  False\u003Cbr>Or()\u003Cbr>\u003Cbr>True ✓ |  |  |\n| ProTo\u003Cbr>Results | Select(racket)\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>Filter(white)\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>Exist()\u003Cbr>| | Select (plate)\u003Cbr>\u003Cbr>\u003Cbr>Relate(to the right, man)\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>Exist()\u003Cbr>\u003Cbr>True ✓ |\n|  | True  False\u003Cbr>Or()\u003Cbr>True ✓\u003Cbr>|  |  |\n\nFigure 2: Visualization of validation results on the GQA dataset (Part 2) . We use the green tick and the red cross to denote right and wrong reasoning results, respectively. Both models predict the correct answer in the first example, and the reasoning traces are the same. For the second example, our model gets the correct answer, but the MMN baseline fails.  \n\n| Program | Goto(2, 3)\u003Cbr>Mine(Rectangle)\u003Cbr>Place(Triangle , 5, 4) If Agent[Triangle] \u003C 3:\u003Cbr>Place(Circle, 8, 7)\u003Cbr>If Agent[Rectangle] \u003C 1 :\u003Cbr>Place(Rectangle , 1 , 8) | Initial\u003Cbr>Specification | |\n| --- | --- | --- | --- |\n| PGA\u003Cbr>Results | \u003Cbr>\u003Cbr>✓ ✘\u003Cbr>If Agent[Triangle] \u003C 3 True  Place(Circle, 8, 7)\u003Cbr>\u003Cbr>◯\u003Cbr>If Agent[Rectangle] \u003C 1  False\u003Cbr>|  |  |\n| ProTo\u003Cbr>Results | True\u003Cbr>\u003Cbr>Mine(Rectangle)\u003Cbr>\u003Cbr>Place (Triangle , 5, 4)\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>If Agent[Triangle] \u003C 3\u003Cbr>\u003Cbr> Place(Circle, 8, 7)\u003Cbr>\u003Cbr>✓\u003Cbr>✓\u003Cbr>✓\u003Cbr>✓\u003Cbr>✓\u003Cbr>If Agent[Rectangle] \u003C 1\u003Cbr>\u003Cbr>\u003Cbr>False\u003Cbr>✓\u003Cbr>Goto(2, 3)\u003Cbr>|  |  |\n\nFigure 3: Visualization of test results on the Minecraft environment in comparison to program-guided agent [81](Part 1) . Due to space limitations, we omit the middle steps for executing routines and directly show the execution results of routines. We use the green tick and the red cross to denote right and wrong reasoning results, respectively. Additionally, we use a red circle to indicate that the agent has failed some routine (therefore, the program execution is failed) . We observe that the ProTo agent can build a bridge to cross a river to place a circle on the other side of the river, but PGA fails to do so.  \n\n|  | While Env[Gold] > 1:\u003Cbr>Mine(Gold) |  | Initial |  |  |\n| --- | --- | --- | --- | --- | --- |\n| Program | If Agent[Gold]\u003C3: |  |  |  |  |\n|  | If Env[Gold] > 1: Mine (Iron) |  | Specification |  |  |\n| PGA\u003Cbr>Results | True\u003Cbr>While Env[Gold] > 1\u003Cbr>\u003Cbr>Mine(Gold)\u003Cbr>If Agent[Gold]\u003C3\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>✓\u003Cbr>If Env[Gold] > 1\u003Cbr>✓ ✓\u003Cbr>\u003Cbr>\u003Cbr>Mine(Iron)\u003Cbr>\u003Cbr>✓\u003Cbr>✓\u003Cbr>\u003Cbr>\u003Cbr>\u003Cbr>✓\u003Cbr>True\u003Cbr>\u003Cbr>True\u003Cbr>\u003Cbr>\u003Cbr>While Env[Gold] > 1 True\u003Cbr>Mine(Gold)\u003Cbr>\u003Cbr>If Agent[Gold]\u003C3\u003Cbr>\u003Cbr>✓\u003Cbr> True\u003Cbr>If Env[Gold] > 1\u003Cbr>\u003Cbr>✓\u003Cbr>True\u003Cbr>\u003Cbr>Mine(Iron)\u003Cbr>\u003Cbr>✓\u003Cbr>\u003Cbr>While Env[Gold] > 1\u003Cbr>\u003Cbr>True\u003Cbr>\u003Cbr>✘\u003Cbr>Mine(Gold)\u003Cbr>\u003Cbr>TIMEOUT! |  |  |  |  |\n| ProTo\u003Cbr>Results | ✓ ✓\u003Cbr> True \u003Cbr>✓\u003Cbr> True\u003Cbr> True   While Env[Gold] > 1 True\u003Cbr>✓ ✓\u003Cbr>\u003Cbr>While Env[Gold] > 1\u003Cbr>Mine(Gold)\u003Cbr>\u003Cbr>✓ ✓ ✓\u003Cbr>If Env[Gold] > 1\u003Cbr>Mine(Iron)\u003Cbr>\u003Cbr>If Agent[Gold]\u003C3 |  |  |  |  |\n|  | Mine(Gold)  If Agent[Gold]\u003C3 TrueIf Env[Gold] > 1  True |  |  |  |  |\n|  | ✓\u003Cbr>\u003Cbr>Mine(Iron) | ✓\u003Cbr>While Env[Gold] > 1  True |  |  | |\n|  | ✓ ✓\u003Cbr>If Agent[Gold]\u003C3 False  While Env[Gold] > 1  False |","cbCair4ezGXIWW7h","https://ap.wps.com/l/cbCair4ezGXIWW7h","pdf",10906636,"English","# Figure 1 Validation Visualization (GQA, Part 1)\n## MMN vs ProTo Reasoning Outcomes\n# Figure 2 Validation Visualization (GQA, Part 2)\n## Correct Answers and Reasoning Trace Comparisons\n# Figure 3 Test Visualization (Minecraft, Part 1)\n## Program-Guided Agent vs ProTo Agent\n# Figure 4 Test Visualization (Minecraft, Part 2)\n## Timeout and Successful Routine Execution\n# Extended Related Work\n## Transformers with Masked Attention\n## Neural Symbolic Learning\n# Program Details\n## Program-guided Visual Reasoning\n## Programs on Two Experimented Tasks","[{\"question\":\"What do the validation visualizations on the GQA dataset compare?\",\"answer\":\"They compare reasoning and answer correctness between the MMN approach and the ProTo model across example questions, using green ticks and red crosses to mark correct and wrong reasoning results.\"},{\"question\":\"How does the ProTo model differ from the MMN baseline on the GQA examples shown?\",\"answer\":\"The figures indicate cases where ProTo predicts the correct answer while MMN fails, and where reasoning traces can differ even when the final answers match.\"},{\"question\":\"What program details are provided for the experimental tasks?\",\"answer\":\"The document includes additional details of the programs used on two tasks, including a section on program-guided visual reasoning and the structure of routines in the GQA dataset.\"}]","NeurIPS 2021 - Proto Program Guided Transformer for Program Guided Tasks - Supplemental | PDF"]