mirror of
https://github.com/boostorg/math.git
synced 2026-02-24 16:12:15 +00:00
130 lines
11 KiB
HTML
130 lines
11 KiB
HTML
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
|
|
<title>Gradient Desccent</title>
|
|
<link rel="stylesheet" href="../../math.css" type="text/css">
|
|
<meta name="generator" content="DocBook XSL Stylesheets Vsnapshot">
|
|
<link rel="home" href="../../index.html" title="Math Toolkit 4.2.1">
|
|
<link rel="up" href="../gd_opt.html" title="Gradient Based Optimizers">
|
|
<link rel="prev" href="introduction.html" title="Introduction">
|
|
<link rel="next" href="nesterov.html" title="Nesterov Gradient Desccent">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
</head>
|
|
<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
|
|
<table cellpadding="2" width="100%"><tr>
|
|
<td valign="top"><img alt="Boost C++ Libraries" width="277" height="86" src="../../../../../../boost.png"></td>
|
|
<td align="center"><a href="../../../../../../index.html">Home</a></td>
|
|
<td align="center"><a href="../../../../../../libs/libraries.htm">Libraries</a></td>
|
|
<td align="center"><a href="http://www.boost.org/users/people.html">People</a></td>
|
|
<td align="center"><a href="http://www.boost.org/users/faq.html">FAQ</a></td>
|
|
<td align="center"><a href="../../../../../../more/index.htm">More</a></td>
|
|
</tr></table>
|
|
<hr>
|
|
<div class="spirit-nav">
|
|
<a accesskey="p" href="introduction.html"><img src="../../../../../../doc/src/images/prev.png" alt="Prev"></a><a accesskey="u" href="../gd_opt.html"><img src="../../../../../../doc/src/images/up.png" alt="Up"></a><a accesskey="h" href="../../index.html"><img src="../../../../../../doc/src/images/home.png" alt="Home"></a><a accesskey="n" href="nesterov.html"><img src="../../../../../../doc/src/images/next.png" alt="Next"></a>
|
|
</div>
|
|
<div class="section">
|
|
<div class="titlepage"><div><div><h3 class="title">
|
|
<a name="math_toolkit.gd_opt.gradient_descent"></a><a class="link" href="gradient_descent.html" title="Gradient Desccent">Gradient Desccent</a>
|
|
</h3></div></div></div>
|
|
<h5>
|
|
<a name="math_toolkit.gd_opt.gradient_descent.h0"></a>
|
|
<span class="phrase"><a name="math_toolkit.gd_opt.gradient_descent.synopsis"></a></span><a class="link" href="gradient_descent.html#math_toolkit.gd_opt.gradient_descent.synopsis">Synopsis</a>
|
|
</h5>
|
|
<pre class="programlisting"><span class="preprocessor">#include</span> <span class="special"><</span><span class="identifier">boost</span><span class="special">/</span><span class="identifier">math</span><span class="special">/</span><span class="identifier">optimization</span><span class="special">/</span><span class="identifier">gradient_descent</span><span class="special">.</span><span class="identifier">hpp</span><span class="special">></span>
|
|
|
|
<span class="keyword">template</span><span class="special"><</span><span class="keyword">typename</span> <span class="identifier">ArgumentContainer</span><span class="special">,</span>
|
|
<span class="keyword">typename</span> <span class="identifier">RealType</span><span class="special">,</span>
|
|
<span class="keyword">class</span> <span class="identifier">Objective</span><span class="special">,</span>
|
|
<span class="keyword">class</span> <span class="identifier">InitializationPolicy</span><span class="special">,</span>
|
|
<span class="keyword">class</span> <span class="identifier">ObjectiveEvalPolicy</span><span class="special">,</span>
|
|
<span class="keyword">class</span> <span class="identifier">GradEvalPolicy</span><span class="special">></span>
|
|
<span class="keyword">class</span> <span class="identifier">gradient_descent</span> <span class="special">{</span>
|
|
<span class="keyword">public</span><span class="special">:</span>
|
|
<span class="keyword">void</span> <span class="identifier">step</span><span class="special">();</span>
|
|
<span class="special">}</span>
|
|
|
|
<span class="comment">/* Convenience overloads */</span>
|
|
<span class="comment">/* make gradient descent by providing
|
|
** objective function
|
|
** variables to optimize over
|
|
** optionally learing rate
|
|
*
|
|
* requires that code is written using boost::math::differentiation::rvar
|
|
*/</span>
|
|
<span class="keyword">template</span><span class="special"><</span><span class="keyword">class</span> <span class="identifier">Objective</span><span class="special">,</span> <span class="keyword">typename</span> <span class="identifier">ArgumentContainer</span><span class="special">,</span> <span class="keyword">typename</span> <span class="identifier">RealType</span><span class="special">></span>
|
|
<span class="keyword">auto</span> <span class="identifier">make_gradient_descent</span><span class="special">(</span><span class="identifier">Objective</span><span class="special">&&</span> <span class="identifier">obj</span><span class="special">,</span> <span class="identifier">ArgumentContainer</span><span class="special">&</span> <span class="identifier">x</span><span class="special">,</span> <span class="identifier">RealType</span> <span class="identifier">lr</span> <span class="special">=</span> <span class="identifier">RealType</span><span class="special">{</span> <span class="number">0.01</span> <span class="special">});</span>
|
|
|
|
<span class="comment">/* make gradient descent by providing
|
|
* objective function
|
|
** variables to optimize over
|
|
** learning rate (not optional)
|
|
** initialization policy
|
|
*
|
|
* requires that code is written using boost::math::differentiation::rvar
|
|
*/</span>
|
|
|
|
<span class="keyword">template</span><span class="special"><</span><span class="keyword">class</span> <span class="identifier">Objective</span><span class="special">,</span> <span class="keyword">typename</span> <span class="identifier">ArgumentContainer</span><span class="special">,</span> <span class="keyword">typename</span> <span class="identifier">RealType</span><span class="special">,</span> <span class="keyword">class</span> <span class="identifier">InitializationPolicy</span><span class="special">></span>
|
|
<span class="keyword">auto</span> <span class="identifier">make_gradient_descent</span><span class="special">(</span><span class="identifier">Objective</span><span class="special">&&</span> <span class="identifier">obj</span><span class="special">,</span>
|
|
<span class="identifier">ArgumentContainer</span><span class="special">&</span> <span class="identifier">x</span><span class="special">,</span>
|
|
<span class="identifier">RealType</span> <span class="identifier">lr</span><span class="special">,</span>
|
|
<span class="identifier">InitializationPolicy</span><span class="special">&&</span> <span class="identifier">ip</span><span class="special">);</span>
|
|
<span class="comment">/* make gradient descent by providing
|
|
** objective function
|
|
** variables to optimize over
|
|
** learning rate (not optional)
|
|
** variable initialization policy
|
|
** objective evaluation policy
|
|
** gradient evaluation policy
|
|
*
|
|
* code does not have to use boost::math::differentiation::rvar
|
|
*/</span>
|
|
<span class="keyword">template</span><span class="special"><</span><span class="keyword">typename</span> <span class="identifier">ArgumentContainer</span><span class="special">,</span>
|
|
<span class="keyword">typename</span> <span class="identifier">RealType</span><span class="special">,</span>
|
|
<span class="keyword">class</span> <span class="identifier">Objective</span><span class="special">,</span>
|
|
<span class="keyword">class</span> <span class="identifier">InitializationPolicy</span><span class="special">,</span>
|
|
<span class="keyword">class</span> <span class="identifier">ObjectiveEvalPolicy</span><span class="special">,</span>
|
|
<span class="keyword">class</span> <span class="identifier">GradEvalPolicy</span><span class="special">></span>
|
|
<span class="keyword">auto</span> <span class="identifier">make_gradient_descent</span><span class="special">(</span><span class="identifier">Objective</span><span class="special">&&</span> <span class="identifier">obj</span><span class="special">,</span>
|
|
<span class="identifier">ArgumentContainer</span><span class="special">&</span> <span class="identifier">x</span><span class="special">,</span>
|
|
<span class="identifier">RealType</span><span class="special">&</span> <span class="identifier">lr</span><span class="special">,</span>
|
|
<span class="identifier">InitializationPolicy</span><span class="special">&&</span> <span class="identifier">ip</span><span class="special">,</span>
|
|
<span class="identifier">ObjectiveEvalPolicy</span><span class="special">&&</span> <span class="identifier">oep</span><span class="special">,</span>
|
|
<span class="identifier">GradEvalPolicy</span><span class="special">&&</span> <span class="identifier">gep</span><span class="special">)</span>
|
|
</pre>
|
|
<p>
|
|
Gradient descent iteratively updates parameters <code class="computeroutput"><span class="identifier">x</span></code>
|
|
in the direction opposite to the gradient of the objective function (minimizing
|
|
the objective).
|
|
</p>
|
|
<pre class="programlisting"><span class="identifier">x</span><span class="special">[</span><span class="identifier">i</span><span class="special">]</span> <span class="special">-=</span> <span class="identifier">lr</span> <span class="special">*</span> <span class="identifier">g</span><span class="special">[</span><span class="identifier">i</span><span class="special">]</span>
|
|
</pre>
|
|
<p>
|
|
where <code class="computeroutput"><span class="identifier">lr</span></code> is a user defined
|
|
learning rate. For a more complete decription of the theoretical principle
|
|
check <a href="https://en.wikipedia.org/wiki/Gradient_descent" target="_top">the wikipedia
|
|
page</a>
|
|
</p>
|
|
<p>
|
|
The implementation delegates: - the initialization of differentiable variables
|
|
to an initialization policy - objective evaluation to an objective evaluation
|
|
policy - the gradient computation to a gradient evaluation policy - the parameter
|
|
updates to an update policy
|
|
</p>
|
|
</div>
|
|
<div class="copyright-footer">Copyright © 2006-2021 Nikhar Agrawal, Anton Bikineev, Matthew Borland,
|
|
Paul A. Bristow, Marco Guazzone, Christopher Kormanyos, Hubert Holin, Bruno
|
|
Lalande, John Maddock, Evan Miller, Jeremy Murphy, Matthew Pulver, Johan Råde,
|
|
Gautam Sewani, Benjamin Sobotta, Nicholas Thompson, Thijs van den Berg, Daryle
|
|
Walker and Xiaogang Zhang<p>
|
|
Distributed under the Boost Software License, Version 1.0. (See accompanying
|
|
file LICENSE_1_0.txt or copy at <a href="http://www.boost.org/LICENSE_1_0.txt" target="_top">http://www.boost.org/LICENSE_1_0.txt</a>)
|
|
</p>
|
|
</div>
|
|
<hr>
|
|
<div class="spirit-nav">
|
|
<a accesskey="p" href="introduction.html"><img src="../../../../../../doc/src/images/prev.png" alt="Prev"></a><a accesskey="u" href="../gd_opt.html"><img src="../../../../../../doc/src/images/up.png" alt="Up"></a><a accesskey="h" href="../../index.html"><img src="../../../../../../doc/src/images/home.png" alt="Home"></a><a accesskey="n" href="nesterov.html"><img src="../../../../../../doc/src/images/next.png" alt="Next"></a>
|
|
</div>
|
|
</body>
|
|
</html>
|