[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"detail-sidebar-cat-0-en-105":3,"doc-seo-394110-105":59,"doc-detail-394110-en":130},{"code":4,"msg":5,"data":6},0,"success",[7,13,18,23,28,33,38,43,48,51,55],{"id":8,"doc_module":4,"doc_module_name":9,"category_name":10,"show_sort_weight":11,"slug":12},1,"Document","Story & Novel",90,"story-novel",{"id":14,"doc_module":4,"doc_module_name":9,"category_name":15,"show_sort_weight":16,"slug":17},2,"Literature",80,"literature",{"id":19,"doc_module":4,"doc_module_name":9,"category_name":20,"show_sort_weight":21,"slug":22},4,"Exam",70,"exam",{"id":24,"doc_module":4,"doc_module_name":9,"category_name":25,"show_sort_weight":26,"slug":27},5,"Comic",60,"comic",{"id":29,"doc_module":4,"doc_module_name":9,"category_name":30,"show_sort_weight":31,"slug":32},6,"Technology",50,"technology",{"id":34,"doc_module":4,"doc_module_name":9,"category_name":35,"show_sort_weight":36,"slug":37},7,"Healthcare",40,"healthcare",{"id":39,"doc_module":4,"doc_module_name":9,"category_name":40,"show_sort_weight":41,"slug":42},8,"Research & Report",30,"research-report",{"id":44,"doc_module":4,"doc_module_name":9,"category_name":45,"show_sort_weight":46,"slug":47},9,"Religion & Spirituality",20,"religion-spirituality",{"id":46,"doc_module":4,"doc_module_name":9,"category_name":49,"show_sort_weight":46,"slug":50},"World Cup","world-cup",{"id":52,"doc_module":4,"doc_module_name":9,"category_name":53,"show_sort_weight":52,"slug":54},10,"Lifestyle","lifestyle",{"id":56,"doc_module":4,"doc_module_name":9,"category_name":57,"show_sort_weight":24,"slug":58},19,"General","general",{"code":4,"msg":60,"data":61},"ok",{"site_id":62,"language":63,"slug":64,"title":65,"keywords":66,"description":67,"schema_data":68,"social_meta":123,"head_meta":125,"extra_data":127,"updated_unix":129},105,"en","iaml-dl-study-guide-week-5-optimisation-and-regularisation","IAML DL-Study Guide - Week 5 - Optimisation and Regularisation","","Week 5 of the IAML DL-Study Guide introduces optimisation and regularisation as essential techniques for training machine learning models. Optimisation focuses on how models learn by minimising error or cost functions, including concepts like model fitting, error manifolds, loss/objective function terminology, and gradient-based methods such as gradient descent, stochastic and mini-batch variants. Regularisation addresses overfitting by adding complexity-penalising terms, emphasizing L2 (Ridge) and L1 (Lasso) regularisation and references for deeper study.",{"@graph":69,"@context":122},[70,84,105],{"@type":71,"itemListElement":72},"BreadcrumbList",[73,77,79,82],{"item":74,"name":75,"@type":76,"position":8},"https://docshare.wps.com","Home","ListItem",{"item":78,"name":9,"@type":76,"position":14},"https://docshare.wps.com/document/",{"item":80,"name":20,"@type":76,"position":81},"https://docshare.wps.com/document/exam/",3,{"item":83,"name":65,"@type":76,"position":19},"https://docshare.wps.com/document/iaml-dl-study-guide-week-5-optimisation-and-regularisation/394110/",{"url":83,"name":65,"@type":85,"image":86,"author":91,"headline":65,"publisher":94,"fileFormat":97,"inLanguage":63,"description":67,"dateModified":98,"datePublished":99,"encodingFormat":97,"isAccessibleForFree":100,"interactionStatistic":101},"DigitalDocument",{"url":87,"@type":88,"width":89,"height":90},"https://docshare.wps.com/thumbnails/iaml-dl-study-guide-week-5-optimisation-and-regularisation/394110.png","ImageObject",300,407,{"name":92,"@type":93},"MrHarris58","Person",{"url":74,"name":95,"@type":96},"DocShare","Organization","application/pdf","2026-10-21","2026-09-26",true,{"@type":102,"interactionType":103,"userInteractionCount":81},"InteractionCounter",{"@type":104},"ViewAction",{"@type":106,"mainEntity":107},"FAQPage",[108,114,118],{"name":109,"@type":110,"acceptedAnswer":111},"What is the difference between optimisation and regularisation in machine learning?","Question",{"text":112,"@type":113},"Optimisation explains how a model learns by minimising an error or cost metric. Regularisation focuses on reducing overfitting by controlling model complexity with added penalty terms.","Answer",{"name":115,"@type":110,"acceptedAnswer":116},"What do loss function, cost function, error function, and objective function refer to?",{"text":117,"@type":113},"They are used as synonyms in this guide, with cost/error/loss typically associated with minimisation, while objective function is more general.",{"name":119,"@type":110,"acceptedAnswer":120},"How do batch gradient descent, stochastic gradient descent, and mini-batch gradient descent differ?",{"text":121,"@type":113},"Batch gradient descent uses the whole dataset to compute gradients. Stochastic gradient descent computes gradients using one randomly chosen sample per epoch. Mini-batch methods use a randomly selected subset and perform batch descent within that subset.","https://schema.org",{"og:url":83,"og:type":124,"og:title":65,"og:site_name":95,"og:description":67},"article",{"robots":126,"canonical":83},"index,follow",{"doc_id":128,"site_id":62},394110,1792032418,{"code":4,"msg":5,"data":131},{"doc_id":128,"user_id":132,"nickname":92,"user_avatar":133,"doc_module":4,"category_id":19,"category_name":20,"doc_title":65,"doc_description":67,"doc_content":134,"file_id":135,"file_url":136,"file_type":137,"file_size":138,"view_count":81,"is_deleted":4,"is_public":8,"is_downloadable":8,"audit_status":8,"page_count":81,"language":139,"language_code":63,"site_id":62,"html_lang":63,"table_of_contents":140,"faqs":141,"seo_title":142,"seo_description":67,"update_tm":143,"read_time":39,"preview_page_count":4},687212988360,"https://ap-avatar.wpscdn.com/davatar_276721f389ce27ea32af1340a28f341c","IAML DL-Study Guide-Week 5  \nSambit Paul, Pavlos Andreadis, Nigel Goddard January 2022  \n1 Introduction  \nWeek 5 introduces the concepts of Optimisation and Regularisation of machine learning models and why they are essential. Optimisation deals with the ‘how’the machine learning model learns, while Regularisation aims to solve the overfitting problem.  \n2 Optimisation and Regularisation  \n2.1 Optimisation  \n• Optimisation is the process of using an error or cost metric to find the most optimal solution to a given problem. The process of optimisation can either be arithmetic, or it can be computed stochastically.  \n• In Machine Learning, we typically use an optimisation process in order to solve the problem of which model parameter assignments make our model best fit the training data (indicated by which parameter assignment minimises our error metric) . The problem is known as model fitting. Of course there is more nuance here, as we do not typically want the model to perfectly fit our training data (consider Generalisation, and Regularisation below) . To read more about the different methods for model fitting, Murphy [2012] Section 8.3 gives a brief introduction to various methods.  \n• An error manifold is the space of all possible values to your objective function in the optimisation problem (assuming you have a criterion that needs to be minimised, such as a metric of how wrong your predictions with the model are) . Given a specific set of samples on which we are comparing, the error manifold will depend on the different possible values our model parameters can take. A brief explanation of error manifolds is provided in this Jupyter Notebook.  \n• Cost function, loss function, error function, and objective function are synonyms, though the first 3 imply that we are dealing with a minimisation problem, while objective function is more general. Typically we are  \nlooking to define a cost function which we need to minimise. Refer to Goodfellow et al. [2016] Section 4.3 to understand the basic idea behind convex optimisation.  \n• To understand and know more about different loss functions in brevity, you can refer to this article. Also, try understanding how each loss function is suited to classification and regression problems. Generally, you might want to introduce further terms to your loss function; this is an engineering problem in and of itself (for example, see Regularisation below) .  \n• Gradient descent has been well explained in Goodfellow et al. [2016] Section 5.2.4 and the key equation behind gradient based optimisation is given in Equation 5.41 .  \n• Learning rate is a hard parameter to tune, and Murphy [2012] Section 8.3.2 on Page 247 aims to explain how it works.  \n• – Batch gradient descent computes the gradient using the whole dataset.  \n– Stochastic gradient descent (SGD) computes the gradient using a single sample randomly chosen over each epoch from the dataset.  \n– Best of both worlds, is to use a mini-batch (set of samples chosen randomly from the dataset) and perform Batch descent over that mini-batch.  \n2.2 Regularisation  \n• The basic idea of regularisation is well articulated in Barber [2012] where he explains it as, ”For most purposes, our interest is not just to find the function that best fits the train data but one that will generalisewell. To control the complexity of the fitted function we may add an extra regularising term to the train error to penalise rapid changes in the output”.  \n• To go through the method of finding weights when L2 regularisation is used in linear regression, you might want to use Bishop [2006] Pg 144 . The equations is w = (ΦT .Φ + λ .I)−1 .ΦT .y. This is Ridge Regression.  \n• A clear explanation of L2 Regularisation (Ridge Regression) is given in Goodfellow et al. [2016] Section 7.1.1 .  \n• Additionally, another important form of regularisation is L1 regularisation or what is known as Lasso Regression. It is useful to find sparse matrices [of weights. lease](of weights. l","cbCaiuweTpUZrrYO","https://ap.wps.com/l/cbCaiuweTpUZrrYO","pdf",131588,"English","# 1 Introduction\n# 2 Optimisation and Regularisation\n## 2.1 Optimisation\n## 2.2 Regularisation\n# References","[{\"question\":\"What is the difference between optimisation and regularisation in machine learning?\",\"answer\":\"Optimisation explains how a model learns by minimising an error or cost metric. Regularisation focuses on reducing overfitting by controlling model complexity with added penalty terms.\"},{\"question\":\"What do loss function, cost function, error function, and objective function refer to?\",\"answer\":\"They are used as synonyms in this guide, with cost/error/loss typically associated with minimisation, while objective function is more general.\"},{\"question\":\"How do batch gradient descent, stochastic gradient descent, and mini-batch gradient descent differ?\",\"answer\":\"Batch gradient descent uses the whole dataset to compute gradients. Stochastic gradient descent computes gradients using one randomly chosen sample per epoch. Mini-batch methods use a randomly selected subset and perform batch descent within that subset.\"}]","IAML DL-Study Guide - Week 5 - Optimisation and Regularisation | PDF",1790398738]