Squashed 'third_party/ceres/' content from commit e51e9b4
Change-Id: I763587619d57e594d3fa158dc3a7fe0b89a1743b
git-subtree-dir: third_party/ceres
git-subtree-split: e51e9b46f6ca88ab8b2266d0e362771db6d98067
diff --git a/internal/ceres/line_search_direction.cc b/internal/ceres/line_search_direction.cc
new file mode 100644
index 0000000..1f9d205
--- /dev/null
+++ b/internal/ceres/line_search_direction.cc
@@ -0,0 +1,372 @@
+// Ceres Solver - A fast non-linear least squares minimizer
+// Copyright 2015 Google Inc. All rights reserved.
+// http://ceres-solver.org/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: sameeragarwal@google.com (Sameer Agarwal)
+
+#include "ceres/line_search_direction.h"
+#include "ceres/line_search_minimizer.h"
+#include "ceres/low_rank_inverse_hessian.h"
+#include "ceres/internal/eigen.h"
+#include "glog/logging.h"
+
+namespace ceres {
+namespace internal {
+
+class SteepestDescent : public LineSearchDirection {
+ public:
+ virtual ~SteepestDescent() {}
+ bool NextDirection(const LineSearchMinimizer::State& previous,
+ const LineSearchMinimizer::State& current,
+ Vector* search_direction) {
+ *search_direction = -current.gradient;
+ return true;
+ }
+};
+
+class NonlinearConjugateGradient : public LineSearchDirection {
+ public:
+ NonlinearConjugateGradient(const NonlinearConjugateGradientType type,
+ const double function_tolerance)
+ : type_(type),
+ function_tolerance_(function_tolerance) {
+ }
+
+ bool NextDirection(const LineSearchMinimizer::State& previous,
+ const LineSearchMinimizer::State& current,
+ Vector* search_direction) {
+ double beta = 0.0;
+ Vector gradient_change;
+ switch (type_) {
+ case FLETCHER_REEVES:
+ beta = current.gradient_squared_norm / previous.gradient_squared_norm;
+ break;
+ case POLAK_RIBIERE:
+ gradient_change = current.gradient - previous.gradient;
+ beta = (current.gradient.dot(gradient_change) /
+ previous.gradient_squared_norm);
+ break;
+ case HESTENES_STIEFEL:
+ gradient_change = current.gradient - previous.gradient;
+ beta = (current.gradient.dot(gradient_change) /
+ previous.search_direction.dot(gradient_change));
+ break;
+ default:
+ LOG(FATAL) << "Unknown nonlinear conjugate gradient type: " << type_;
+ }
+
+ *search_direction = -current.gradient + beta * previous.search_direction;
+ const double directional_derivative =
+ current.gradient.dot(*search_direction);
+ if (directional_derivative > -function_tolerance_) {
+ LOG(WARNING) << "Restarting non-linear conjugate gradients: "
+ << directional_derivative;
+ *search_direction = -current.gradient;
+ }
+
+ return true;
+ }
+
+ private:
+ const NonlinearConjugateGradientType type_;
+ const double function_tolerance_;
+};
+
+class LBFGS : public LineSearchDirection {
+ public:
+ LBFGS(const int num_parameters,
+ const int max_lbfgs_rank,
+ const bool use_approximate_eigenvalue_bfgs_scaling)
+ : low_rank_inverse_hessian_(num_parameters,
+ max_lbfgs_rank,
+ use_approximate_eigenvalue_bfgs_scaling),
+ is_positive_definite_(true) {}
+
+ virtual ~LBFGS() {}
+
+ bool NextDirection(const LineSearchMinimizer::State& previous,
+ const LineSearchMinimizer::State& current,
+ Vector* search_direction) {
+ CHECK(is_positive_definite_)
+ << "Ceres bug: NextDirection() called on L-BFGS after inverse Hessian "
+ << "approximation has become indefinite, please contact the "
+ << "developers!";
+
+ low_rank_inverse_hessian_.Update(
+ previous.search_direction * previous.step_size,
+ current.gradient - previous.gradient);
+
+ search_direction->setZero();
+ low_rank_inverse_hessian_.RightMultiply(current.gradient.data(),
+ search_direction->data());
+ *search_direction *= -1.0;
+
+ if (search_direction->dot(current.gradient) >= 0.0) {
+ LOG(WARNING) << "Numerical failure in L-BFGS update: inverse Hessian "
+ << "approximation is not positive definite, and thus "
+ << "initial gradient for search direction is positive: "
+ << search_direction->dot(current.gradient);
+ is_positive_definite_ = false;
+ return false;
+ }
+
+ return true;
+ }
+
+ private:
+ LowRankInverseHessian low_rank_inverse_hessian_;
+ bool is_positive_definite_;
+};
+
+class BFGS : public LineSearchDirection {
+ public:
+ BFGS(const int num_parameters,
+ const bool use_approximate_eigenvalue_scaling)
+ : num_parameters_(num_parameters),
+ use_approximate_eigenvalue_scaling_(use_approximate_eigenvalue_scaling),
+ initialized_(false),
+ is_positive_definite_(true) {
+ LOG_IF(WARNING, num_parameters_ >= 1e3)
+ << "BFGS line search being created with: " << num_parameters_
+ << " parameters, this will allocate a dense approximate inverse Hessian"
+ << " of size: " << num_parameters_ << " x " << num_parameters_
+ << ", consider using the L-BFGS memory-efficient line search direction "
+ << "instead.";
+ // Construct inverse_hessian_ after logging warning about size s.t. if the
+ // allocation crashes us, the log will highlight what the issue likely was.
+ inverse_hessian_ = Matrix::Identity(num_parameters, num_parameters);
+ }
+
+ virtual ~BFGS() {}
+
+ bool NextDirection(const LineSearchMinimizer::State& previous,
+ const LineSearchMinimizer::State& current,
+ Vector* search_direction) {
+ CHECK(is_positive_definite_)
+ << "Ceres bug: NextDirection() called on BFGS after inverse Hessian "
+ << "approximation has become indefinite, please contact the "
+ << "developers!";
+
+ const Vector delta_x = previous.search_direction * previous.step_size;
+ const Vector delta_gradient = current.gradient - previous.gradient;
+ const double delta_x_dot_delta_gradient = delta_x.dot(delta_gradient);
+
+ // The (L)BFGS algorithm explicitly requires that the secant equation:
+ //
+ // B_{k+1} * s_k = y_k
+ //
+ // Is satisfied at each iteration, where B_{k+1} is the approximated
+ // Hessian at the k+1-th iteration, s_k = (x_{k+1} - x_{k}) and
+ // y_k = (grad_{k+1} - grad_{k}). As the approximated Hessian must be
+ // positive definite, this is equivalent to the condition:
+ //
+ // s_k^T * y_k > 0 [s_k^T * B_{k+1} * s_k = s_k^T * y_k > 0]
+ //
+ // This condition would always be satisfied if the function was strictly
+ // convex, alternatively, it is always satisfied provided that a Wolfe line
+ // search is used (even if the function is not strictly convex). See [1]
+ // (p138) for a proof.
+ //
+ // Although Ceres will always use a Wolfe line search when using (L)BFGS,
+ // practical implementation considerations mean that the line search
+ // may return a point that satisfies only the Armijo condition, and thus
+ // could violate the Secant equation. As such, we will only use a step
+ // to update the Hessian approximation if:
+ //
+ // s_k^T * y_k > tolerance
+ //
+ // It is important that tolerance is very small (and >=0), as otherwise we
+ // might skip the update too often and fail to capture important curvature
+ // information in the Hessian. For example going from 1e-10 -> 1e-14
+ // improves the NIST benchmark score from 43/54 to 53/54.
+ //
+ // [1] Nocedal J, Wright S, Numerical Optimization, 2nd Ed. Springer, 1999.
+ //
+ // TODO(alexs.mac): Consider using Damped BFGS update instead of
+ // skipping update.
+ const double kBFGSSecantConditionHessianUpdateTolerance = 1e-14;
+ if (delta_x_dot_delta_gradient <=
+ kBFGSSecantConditionHessianUpdateTolerance) {
+ VLOG(2) << "Skipping BFGS Update, delta_x_dot_delta_gradient too "
+ << "small: " << delta_x_dot_delta_gradient << ", tolerance: "
+ << kBFGSSecantConditionHessianUpdateTolerance
+ << " (Secant condition).";
+ } else {
+ // Update dense inverse Hessian approximation.
+
+ if (!initialized_ && use_approximate_eigenvalue_scaling_) {
+ // Rescale the initial inverse Hessian approximation (H_0) to be
+ // iteratively updated so that it is of similar 'size' to the true
+ // inverse Hessian at the start point. As shown in [1]:
+ //
+ // \gamma = (delta_gradient_{0}' * delta_x_{0}) /
+ // (delta_gradient_{0}' * delta_gradient_{0})
+ //
+ // Satisfies:
+ //
+ // (1 / \lambda_m) <= \gamma <= (1 / \lambda_1)
+ //
+ // Where \lambda_1 & \lambda_m are the smallest and largest eigenvalues
+ // of the true initial Hessian (not the inverse) respectively. Thus,
+ // \gamma is an approximate eigenvalue of the true inverse Hessian, and
+ // choosing: H_0 = I * \gamma will yield a starting point that has a
+ // similar scale to the true inverse Hessian. This technique is widely
+ // reported to often improve convergence, however this is not
+ // universally true, particularly if there are errors in the initial
+ // gradients, or if there are significant differences in the sensitivity
+ // of the problem to the parameters (i.e. the range of the magnitudes of
+ // the components of the gradient is large).
+ //
+ // The original origin of this rescaling trick is somewhat unclear, the
+ // earliest reference appears to be Oren [1], however it is widely
+ // discussed without specific attributation in various texts including
+ // [2] (p143).
+ //
+ // [1] Oren S.S., Self-scaling variable metric (SSVM) algorithms
+ // Part II: Implementation and experiments, Management Science,
+ // 20(5), 863-874, 1974.
+ // [2] Nocedal J., Wright S., Numerical Optimization, Springer, 1999.
+ const double approximate_eigenvalue_scale =
+ delta_x_dot_delta_gradient / delta_gradient.dot(delta_gradient);
+ inverse_hessian_ *= approximate_eigenvalue_scale;
+
+ VLOG(4) << "Applying approximate_eigenvalue_scale: "
+ << approximate_eigenvalue_scale << " to initial inverse "
+ << "Hessian approximation.";
+ }
+ initialized_ = true;
+
+ // Efficient O(num_parameters^2) BFGS update [2].
+ //
+ // Starting from dense BFGS update detailed in Nocedal [2] p140/177 and
+ // using: y_k = delta_gradient, s_k = delta_x:
+ //
+ // \rho_k = 1.0 / (s_k' * y_k)
+ // V_k = I - \rho_k * y_k * s_k'
+ // H_k = (V_k' * H_{k-1} * V_k) + (\rho_k * s_k * s_k')
+ //
+ // This update involves matrix, matrix products which naively O(N^3),
+ // however we can exploit our knowledge that H_k is positive definite
+ // and thus by defn. symmetric to reduce the cost of the update:
+ //
+ // Expanding the update above yields:
+ //
+ // H_k = H_{k-1} +
+ // \rho_k * ( (1.0 + \rho_k * y_k' * H_k * y_k) * s_k * s_k' -
+ // (s_k * y_k' * H_k + H_k * y_k * s_k') )
+ //
+ // Using: A = (s_k * y_k' * H_k), and the knowledge that H_k = H_k', the
+ // last term simplifies to (A + A'). Note that although A is not symmetric
+ // (A + A') is symmetric. For ease of construction we also define
+ // B = (1 + \rho_k * y_k' * H_k * y_k) * s_k * s_k', which is by defn
+ // symmetric due to construction from: s_k * s_k'.
+ //
+ // Now we can write the BFGS update as:
+ //
+ // H_k = H_{k-1} + \rho_k * (B - (A + A'))
+
+ // For efficiency, as H_k is by defn. symmetric, we will only maintain the
+ // *lower* triangle of H_k (and all intermediary terms).
+
+ const double rho_k = 1.0 / delta_x_dot_delta_gradient;
+
+ // Calculate: A = s_k * y_k' * H_k
+ Matrix A = delta_x * (delta_gradient.transpose() *
+ inverse_hessian_.selfadjointView<Eigen::Lower>());
+
+ // Calculate scalar: (1 + \rho_k * y_k' * H_k * y_k)
+ const double delta_x_times_delta_x_transpose_scale_factor =
+ (1.0 + (rho_k * delta_gradient.transpose() *
+ inverse_hessian_.selfadjointView<Eigen::Lower>() *
+ delta_gradient));
+ // Calculate: B = (1 + \rho_k * y_k' * H_k * y_k) * s_k * s_k'
+ Matrix B = Matrix::Zero(num_parameters_, num_parameters_);
+ B.selfadjointView<Eigen::Lower>().
+ rankUpdate(delta_x, delta_x_times_delta_x_transpose_scale_factor);
+
+ // Finally, update inverse Hessian approximation according to:
+ // H_k = H_{k-1} + \rho_k * (B - (A + A')). Note that (A + A') is
+ // symmetric, even though A is not.
+ inverse_hessian_.triangularView<Eigen::Lower>() +=
+ rho_k * (B - A - A.transpose());
+ }
+
+ *search_direction =
+ inverse_hessian_.selfadjointView<Eigen::Lower>() *
+ (-1.0 * current.gradient);
+
+ if (search_direction->dot(current.gradient) >= 0.0) {
+ LOG(WARNING) << "Numerical failure in BFGS update: inverse Hessian "
+ << "approximation is not positive definite, and thus "
+ << "initial gradient for search direction is positive: "
+ << search_direction->dot(current.gradient);
+ is_positive_definite_ = false;
+ return false;
+ }
+
+ return true;
+ }
+
+ private:
+ const int num_parameters_;
+ const bool use_approximate_eigenvalue_scaling_;
+ Matrix inverse_hessian_;
+ bool initialized_;
+ bool is_positive_definite_;
+};
+
+LineSearchDirection*
+LineSearchDirection::Create(const LineSearchDirection::Options& options) {
+ if (options.type == STEEPEST_DESCENT) {
+ return new SteepestDescent;
+ }
+
+ if (options.type == NONLINEAR_CONJUGATE_GRADIENT) {
+ return new NonlinearConjugateGradient(
+ options.nonlinear_conjugate_gradient_type,
+ options.function_tolerance);
+ }
+
+ if (options.type == ceres::LBFGS) {
+ return new ceres::internal::LBFGS(
+ options.num_parameters,
+ options.max_lbfgs_rank,
+ options.use_approximate_eigenvalue_bfgs_scaling);
+ }
+
+ if (options.type == ceres::BFGS) {
+ return new ceres::internal::BFGS(
+ options.num_parameters,
+ options.use_approximate_eigenvalue_bfgs_scaling);
+ }
+
+ LOG(ERROR) << "Unknown line search direction type: " << options.type;
+ return NULL;
+}
+
+} // namespace internal
+} // namespace ceres