[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-515044-105":59,"doc-detail-515044-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","supplemental-materials-fixing-a-broken-elbo","Supplemental Materials - Fixing a Broken ELBO","","Supplemental materials analyze broken ELBO behavior in variational autoencoder model families, focusing on rate-distortion tradeoffs, reconstruction and generation quality, and how semantic versus syntactic encoders emerge under different architectures. Additional experiments cover Static MNIST and OMNIGLOT, showing smooth interpolation between autodecoding and autoencoding by tuning the latent information parameter, with KL annealing enabling finite rates. The document also derives inequalities for generative mutual information using alternate probabilistic paths and variational bounds.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":40,"@type":76,"position":81},"https://docshare.wps.com/document/research-report/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/supplemental-materials-fixing-a-broken-elbo/515044/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/supplemental-materials-fixing-a-broken-elbo/515044.png","ImageObject",300,407,{"name":92,"@type":93},"pop out","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-11-15","2026-10-06",true,{"@type":102,"interactionType":103,"userInteractionCount":52},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"How do the figures on Static MNIST characterize different VAE behaviors?","Question",{"text":112,"@type":113},"They contrast syntactic encoders and autodecoders by rate-distortion tradeoffs and qualitative sample variance. The discussion notes that none of the 12 VAEs acts as a semantic encoder and that achieving semantics is difficult when the tuning parameter equals 1.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What do the OMNIGLOT experiments show about powerful versus weaker decoder models?",{"text":117,"@type":113},"Powerful decoder models dominate weaker decoders and naturally operate at very low rates. Finite rates are obtained via KL annealing, and the best ELBO is achieved by a ++-model at a specific latent parameter value.",{"name":119,"@type":110,"acceptedAnswer":120},"What is the generative mutual information section trying to prove?",{"text":121,"@type":113},"It derives an inequality relating three functionals of distributions by defining a generative path through an encoder, a decoder, and a marginal density. Variational lower and upper bounds are then established for the mutual information along this path.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},515044,1791397049,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":39,"category_name":40,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":52,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":139,"language":140,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":141,"faqs":142,"seo_title":143,"seo_description":67,"update_tm":144,"read_time":145,"preview_page_count":19,"preview_urls":146},962090760408,"https://ap-avatar.wpscdn.com/davatar_085a072bc5b1113ac321206ff7593b45","Supplemental Materials: Fixing a Broken ELBO  \nA. More results on Static MNIST  \n(a) (reconstructions)  \n(b) (generations)  \nFigure 5. Traditional VAE behaviors of all model families. Note the clear separation between syntactic encoders and autodecoders, both in terms of the rate-distortion tradeoff, and in qualitative terms of sample variance. Also note that none of the 12 VAEs is a semantic encoder. Semantic encoding seems difﬁcult to achieve at 􀀌 = 1 .  \nFigure 6 illustrates that many different architectures can participate in the optimal frontier and that we can achieve a smooth variation between the pure autodecoding models and models that encode more and more semantic and syntactic information. On the left, we see three syntactic encoders, which do a good job of capturing both the content of the digit and its style, while having variance in the decodings that seem to capture the sampling noise. On the right, we have six clear autodecoders, with very low rate and very high variance in the reconstructed or generated digit. In between are three semantic encoders, capturing the class of each digit, but showing a wide range of decoded style variation, which corresponds to the syntax of MNIST digits. Finally, between the syntactic encoders and semantic encoders lies a modeling failure, in which a weak encoder and marginal are paired with a strong decoder. The rate is sufﬁciently high for the decoder to reconstruct a good amount of the semantic and syntactic information, but it appears to have failed to learn to distinguish between the two.  \nB. Results on OMNIGLOT  \nFigure 7 plots the RD curve for various models ﬁt to the Omniglot dataset (Lake et al., 2015), in the same form as the MNIST results in Figure 3. Here we explored 􀀌s for the powerful decoder models ranging from 1.1 to 0.1, and 􀀌s of 0.9, 1.0, and 1.1 for the weaker decoder models. After these runs, it was clear that much of the frontier was missing, so given the already dominating performance of the powerful decoder models, those were additionally run for 􀀌s of 0:2; 0:3; 0:4; 0:5; 0:6; 0:7; 0:8 to ﬁll in the diagram.  \nOn Omniglot, the powerful decoder models dominate over the weaker decoder models. The powerful decoder models with their autoregressive form most naturally sit at very low rates. We were able to obtain ﬁnite rates by means of KL annealing. Our best achieved ELBO was at-90.37 nats, set by the ++-model with 􀀌 = 1:0 and KL annealing. This model obtains R = 0:77; D = 89:60; ELBO = 􀀀90:37 and is nearly auto-decoding. We found 14 models with ELBOs below 91.2 nats ranging in rates from 0.0074 nats to 10.92 nats.  \n(a) (reconstructions)  \n(b) (generations)  \nFigure 6. Exploring the frontier. Here we show the reconstructions (a) and generated samples (b) from a collection of runs that all lie on the frontier of realizable rate distortion tradeoffs.  \nSimilar to Figure 4 in Figure 8 we show sample reconstruction and generated images from the same ”-+v” model family trained with KL annealing but at various 􀀌s. Just like in the MNIST case, this demonstrates that we can smoothly interpolate between auto-decoding and auto-encoding behavior in a single model family, simply by adjusting the 􀀌 value.  \nC. Generative mutual information  \nGiven any four distributions: p􀀃 (x)– a density over some data space X , e (z jx)– a stochastic map from that data to a new representational space Z , d (xj z)– a stochastic map in the reverse direction from Z to X, and m(z)– some density in the Z space; we were able to ﬁnd an inequality relating three functionals of these densities that must always hold. We found this inequality by deriving upper and lower bounds on the mutual information in the joint density deﬁned by the natural representational path through the four distributions, pe (x; z) = p􀀃 (x)e (z jx) . Doing so naturally made us consider the existence of two other distributions d(xj z) and m(z) . Let's consider the mutual information along this new generative path.  \npd (x;","cbCaie7lnMaM9VUu","https://ap.wps.com/l/cbCaie7lnMaM9VUu","pdf",1712848,11,"English","# Supplemental Materials\n## Fixing a Broken ELBO\n## More results on Static MNIST\n## Results on OMNIGLOT\n## Generative mutual information","[{\"question\":\"How do the figures on Static MNIST characterize different VAE behaviors?\",\"answer\":\"They contrast syntactic encoders and autodecoders by rate-distortion tradeoffs and qualitative sample variance. The discussion notes that none of the 12 VAEs acts as a semantic encoder and that achieving semantics is difficult when the tuning parameter equals 1.\"},{\"question\":\"What do the OMNIGLOT experiments show about powerful versus weaker decoder models?\",\"answer\":\"Powerful decoder models dominate weaker decoders and naturally operate at very low rates. Finite rates are obtained via KL annealing, and the best ELBO is achieved by a ++-model at a specific latent parameter value.\"},{\"question\":\"What is the generative mutual information section trying to prove?\",\"answer\":\"It derives an inequality relating three functionals of distributions by defining a generative path through an encoder, a decoder, and a marginal density. Variational lower and upper bounds are then established for the mutual information along this path.\"}]","Supplemental Materials - Fixing a Broken ELBO | PDF",1791305632,28,[147,151,155,159],{"page_no":8,"url":148,"width":149,"height":150},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/16a4b2d09fce629a8cb6f5470c56f44f.webp",983,1272,{"page_no":14,"url":152,"width":153,"height":154},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/e2371587b53c1113c977fbe597ee2132.webp",982,1270,{"page_no":81,"url":156,"width":157,"height":158},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/1533fb5ba08e806b1ddccd5ef5e9b587.webp",981,1269,{"page_no":19,"url":160,"width":161,"height":162},"https://res-academy.cache.wpscdn.com/upload/kdocshare/2026-11-17/310551e1d85deb4e2b25fc7976aa9eac.webp",980,1268]