+
+
+
+ Gradient based optimizers are algorithms that use the gradient of a funciton
+ to iteratively find locally extreme points of functions over a set of parameters.
+ This sections provides a description of a set of gradient optimizers. The optimizers
+ are written with boost::math::differentiation::reverse_mode::rvar in
+ mind, however if a way to evaluate the funciton and its gradient is provided,
+ the optimizers should work in exactly the same way.
+
+
+
+
+
#include <boost/math/optimization/gradient_descent.hpp>
+
+template<typename ArgumentContainer,
+ typename RealType,
+ class Objective,
+ class InitializationPolicy,
+ class ObjectiveEvalPolicy,
+ class GradEvalPolicy>
+class gradient_descent {
+ public:
+ void step();
+}
+
+
+
+template<class Objective, typename ArgumentContainer, typename RealType>
+auto make_gradient_descent(Objective&& obj, ArgumentContainer& x, RealType lr = RealType{ 0.01 });
+
+
+
+template<class Objective, typename ArgumentContainer, typename RealType, class InitializationPolicy>
+ auto make_gradient_descent(Objective&& obj,
+ ArgumentContainer& x,
+ RealType lr,
+ InitializationPolicy&& ip);
+
+template<typename ArgumentContainer,
+ typename RealType,
+ class Objective,
+ class InitializationPolicy,
+ class ObjectiveEvalPolicy,
+ class GradEvalPolicy>
+auto make_gradient_descent(Objective&& obj,
+ ArgumentContainer& x,
+ RealType& lr,
+ InitializationPolicy&& ip,
+ ObjectiveEvalPolicy&& oep,
+ GradEvalPolicy&& gep)
+
+
+ Gradient descent iteratively updates parameters x
+ in the direction opposite to the gradient of the objective function (minimizing
+ the objective).
+
+
x[i] -= lr * g[i]
+
+
+ where lr is a user defined
+ learning rate. For a more complete decription of the theoretical principle
+ check the wikipedia
+ page
+
+
+ The implementation delegates: - the initialization of differentiable variables
+ to an initialization policy - objective evaluation to an objective evaluation
+ policy - the gradient computation to a gradient evaluation policy - the parameter
+ updates to an update policy
+
+
+
+