Initial Commit

include/Eigen/src/Core/ArithmeticSequence.h

@@ -0,0 +1,239 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_ARITHMETIC_SEQUENCE_H
+#define EIGEN_ARITHMETIC_SEQUENCE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+// Helper to cleanup the type of the increment:
+template <typename T>
+struct cleanup_seq_incr {
+  typedef typename cleanup_index_type<T, DynamicIndex>::type type;
+};
+
+}  // namespace internal
+
+//--------------------------------------------------------------------------------
+// seq(first,last,incr) and seqN(first,size,incr)
+//--------------------------------------------------------------------------------
+
+template <typename FirstType = Index, typename SizeType = Index, typename IncrType = internal::FixedInt<1> >
+class ArithmeticSequence;
+
+template <typename FirstType, typename SizeType, typename IncrType>
+ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,
+                   typename internal::cleanup_index_type<SizeType>::type,
+                   typename internal::cleanup_seq_incr<IncrType>::type>
+seqN(FirstType first, SizeType size, IncrType incr);
+
+/** \class ArithmeticSequence
+ * \ingroup Core_Module
+ *
+ * This class represents an arithmetic progression \f$ a_0, a_1, a_2, ..., a_{n-1}\f$ defined by
+ * its \em first value \f$ a_0 \f$, its \em size (aka length) \em n, and the \em increment (aka stride)
+ * that is equal to \f$ a_{i+1}-a_{i}\f$ for any \em i.
+ *
+ * It is internally used as the return type of the Eigen::seq and Eigen::seqN functions, and as the input arguments
+ * of DenseBase::operator()(const RowIndices&, const ColIndices&), and most of the time this is the
+ * only way it is used.
+ *
+ * \tparam FirstType type of the first element, usually an Index,
+ *                   but internally it can be a symbolic expression
+ * \tparam SizeType type representing the size of the sequence, usually an Index
+ *                  or a compile time integral constant. Internally, it can also be a symbolic expression
+ * \tparam IncrType type of the increment, can be a runtime Index, or a compile time integral constant (default is
+ * compile-time 1)
+ *
+ * \sa Eigen::seq, Eigen::seqN, DenseBase::operator()(const RowIndices&, const ColIndices&), class IndexedView
+ */
+template <typename FirstType, typename SizeType, typename IncrType>
+class ArithmeticSequence {
+ public:
+  constexpr ArithmeticSequence() = default;
+  constexpr ArithmeticSequence(FirstType first, SizeType size) : m_first(first), m_size(size) {}
+  constexpr ArithmeticSequence(FirstType first, SizeType size, IncrType incr)
+      : m_first(first), m_size(size), m_incr(incr) {}
+
+  enum {
+    // SizeAtCompileTime = internal::get_fixed_value<SizeType>::value,
+    IncrAtCompileTime = internal::get_fixed_value<IncrType, DynamicIndex>::value
+  };
+
+  /** \returns the size, i.e., number of elements, of the sequence */
+  constexpr Index size() const { return m_size; }
+
+  /** \returns the first element \f$ a_0 \f$ in the sequence */
+  constexpr Index first() const { return m_first; }
+
+  /** \returns the value \f$ a_i \f$ at index \a i in the sequence. */
+  constexpr Index operator[](Index i) const { return m_first + i * m_incr; }
+
+  constexpr const FirstType& firstObject() const { return m_first; }
+  constexpr const SizeType& sizeObject() const { return m_size; }
+  constexpr const IncrType& incrObject() const { return m_incr; }
+
+ protected:
+  FirstType m_first;
+  SizeType m_size;
+  IncrType m_incr;
+
+ public:
+  constexpr auto reverse() const -> decltype(Eigen::seqN(m_first + (m_size + fix<-1>()) * m_incr, m_size, -m_incr)) {
+    return seqN(m_first + (m_size + fix<-1>()) * m_incr, m_size, -m_incr);
+  }
+};
+
+/** \returns an ArithmeticSequence starting at \a first, of length \a size, and increment \a incr
+ *
+ * \sa seqN(FirstType,SizeType), seq(FirstType,LastType,IncrType) */
+template <typename FirstType, typename SizeType, typename IncrType>
+ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,
+                   typename internal::cleanup_index_type<SizeType>::type,
+                   typename internal::cleanup_seq_incr<IncrType>::type>
+seqN(FirstType first, SizeType size, IncrType incr) {
+  return ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,
+                            typename internal::cleanup_index_type<SizeType>::type,
+                            typename internal::cleanup_seq_incr<IncrType>::type>(first, size, incr);
+}
+
+/** \returns an ArithmeticSequence starting at \a first, of length \a size, and unit increment
+ *
+ * \sa seqN(FirstType,SizeType,IncrType), seq(FirstType,LastType) */
+template <typename FirstType, typename SizeType>
+ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,
+                   typename internal::cleanup_index_type<SizeType>::type>
+seqN(FirstType first, SizeType size) {
+  return ArithmeticSequence<typename internal::cleanup_index_type<FirstType>::type,
+                            typename internal::cleanup_index_type<SizeType>::type>(first, size);
+}
+
+#ifdef EIGEN_PARSED_BY_DOXYGEN
+
+/** \returns an ArithmeticSequence starting at \a f, up (or down) to \a l, and with positive (or negative) increment \a
+ * incr
+ *
+ * It is essentially an alias to:
+ * \code
+ * seqN(f, (l-f+incr)/incr, incr);
+ * \endcode
+ *
+ * \sa seqN(FirstType,SizeType,IncrType), seq(FirstType,LastType)
+ */
+template <typename FirstType, typename LastType, typename IncrType>
+auto seq(FirstType f, LastType l, IncrType incr);
+
+/** \returns an ArithmeticSequence starting at \a f, up (or down) to \a l, and unit increment
+ *
+ * It is essentially an alias to:
+ * \code
+ * seqN(f,l-f+1);
+ * \endcode
+ *
+ * \sa seqN(FirstType,SizeType), seq(FirstType,LastType,IncrType)
+ */
+template <typename FirstType, typename LastType>
+auto seq(FirstType f, LastType l);
+
+#else  // EIGEN_PARSED_BY_DOXYGEN
+
+template <typename FirstType, typename LastType>
+auto seq(FirstType f, LastType l)
+    -> decltype(seqN(typename internal::cleanup_index_type<FirstType>::type(f),
+                     (typename internal::cleanup_index_type<LastType>::type(l) -
+                      typename internal::cleanup_index_type<FirstType>::type(f) + fix<1>()))) {
+  return seqN(typename internal::cleanup_index_type<FirstType>::type(f),
+              (typename internal::cleanup_index_type<LastType>::type(l) -
+               typename internal::cleanup_index_type<FirstType>::type(f) + fix<1>()));
+}
+
+template <typename FirstType, typename LastType, typename IncrType>
+auto seq(FirstType f, LastType l, IncrType incr)
+    -> decltype(seqN(typename internal::cleanup_index_type<FirstType>::type(f),
+                     (typename internal::cleanup_index_type<LastType>::type(l) -
+                      typename internal::cleanup_index_type<FirstType>::type(f) +
+                      typename internal::cleanup_seq_incr<IncrType>::type(incr)) /
+                         typename internal::cleanup_seq_incr<IncrType>::type(incr),
+                     typename internal::cleanup_seq_incr<IncrType>::type(incr))) {
+  typedef typename internal::cleanup_seq_incr<IncrType>::type CleanedIncrType;
+  return seqN(typename internal::cleanup_index_type<FirstType>::type(f),
+              (typename internal::cleanup_index_type<LastType>::type(l) -
+               typename internal::cleanup_index_type<FirstType>::type(f) + CleanedIncrType(incr)) /
+                  CleanedIncrType(incr),
+              CleanedIncrType(incr));
+}
+
+#endif  // EIGEN_PARSED_BY_DOXYGEN
+
+namespace placeholders {
+
+/** \cpp11
+ * \returns a symbolic ArithmeticSequence representing the last \a size elements with increment \a incr.
+ *
+ * It is a shortcut for: \code seqN(last-(size-fix<1>)*incr, size, incr) \endcode
+ *
+ * \sa lastN(SizeType), seqN(FirstType,SizeType), seq(FirstType,LastType,IncrType) */
+template <typename SizeType, typename IncrType>
+auto lastN(SizeType size, IncrType incr)
+    -> decltype(seqN(Eigen::placeholders::last - (size - fix<1>()) * incr, size, incr)) {
+  return seqN(Eigen::placeholders::last - (size - fix<1>()) * incr, size, incr);
+}
+
+/** \cpp11
+ * \returns a symbolic ArithmeticSequence representing the last \a size elements with a unit increment.
+ *
+ *  It is a shortcut for: \code seq(last+fix<1>-size, last) \endcode
+ *
+ * \sa lastN(SizeType,IncrType, seqN(FirstType,SizeType), seq(FirstType,LastType) */
+template <typename SizeType>
+auto lastN(SizeType size) -> decltype(seqN(Eigen::placeholders::last + fix<1>() - size, size)) {
+  return seqN(Eigen::placeholders::last + fix<1>() - size, size);
+}
+
+}  // namespace placeholders
+
+/** \namespace Eigen::indexing
+  * \ingroup Core_Module
+  *
+  * The sole purpose of this namespace is to be able to import all functions
+  * and symbols that are expected to be used within operator() for indexing
+  * and slicing. If you already imported the whole Eigen namespace:
+  * \code using namespace Eigen; \endcode
+  * then you are already all set. Otherwise, if you don't want/cannot import
+  * the whole Eigen namespace, the following line:
+  * \code using namespace Eigen::indexing; \endcode
+  * is equivalent to:
+  * \code
+  using Eigen::fix;
+  using Eigen::seq;
+  using Eigen::seqN;
+  using Eigen::placeholders::all;
+  using Eigen::placeholders::last;
+  using Eigen::placeholders::lastN;  // c++11 only
+  using Eigen::placeholders::lastp1;
+  \endcode
+  */
+namespace indexing {
+using Eigen::fix;
+using Eigen::seq;
+using Eigen::seqN;
+using Eigen::placeholders::all;
+using Eigen::placeholders::last;
+using Eigen::placeholders::lastN;
+using Eigen::placeholders::lastp1;
+}  // namespace indexing
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_ARITHMETIC_SEQUENCE_H

include/Eigen/src/Core/Array.h

@@ -0,0 +1,369 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_ARRAY_H
+#define EIGEN_ARRAY_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename Scalar_, int Rows_, int Cols_, int Options_, int MaxRows_, int MaxCols_>
+struct traits<Array<Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_>>
+    : traits<Matrix<Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_>> {
+  typedef ArrayXpr XprKind;
+  typedef ArrayBase<Array<Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_>> XprBase;
+};
+}  // namespace internal
+
+/** \class Array
+ * \ingroup Core_Module
+ *
+ * \brief General-purpose arrays with easy API for coefficient-wise operations
+ *
+ * The %Array class is very similar to the Matrix class. It provides
+ * general-purpose one- and two-dimensional arrays. The difference between the
+ * %Array and the %Matrix class is primarily in the API: the API for the
+ * %Array class provides easy access to coefficient-wise operations, while the
+ * API for the %Matrix class provides easy access to linear-algebra
+ * operations.
+ *
+ * See documentation of class Matrix for detailed information on the template parameters
+ * storage layout.
+ *
+ * This class can be extended with the help of the plugin mechanism described on the page
+ * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_ARRAY_PLUGIN.
+ *
+ * \sa \blank \ref TutorialArrayClass, \ref TopicClassHierarchy
+ */
+template <typename Scalar_, int Rows_, int Cols_, int Options_, int MaxRows_, int MaxCols_>
+class Array : public PlainObjectBase<Array<Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_>> {
+ public:
+  typedef PlainObjectBase<Array> Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Array)
+
+  enum { Options = Options_ };
+  typedef typename Base::PlainObject PlainObject;
+
+ protected:
+  template <typename Derived, typename OtherDerived, bool IsVector>
+  friend struct internal::conservative_resize_like_impl;
+
+  using Base::m_storage;
+
+ public:
+  using Base::base;
+  using Base::coeff;
+  using Base::coeffRef;
+
+  /**
+   * The usage of
+   *   using Base::operator=;
+   * fails on MSVC. Since the code below is working with GCC and MSVC, we skipped
+   * the usage of 'using'. This should be done only for operator=.
+   */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array& operator=(const EigenBase<OtherDerived>& other) {
+    return Base::operator=(other);
+  }
+
+  /** Set all the entries to \a value.
+   * \sa DenseBase::setConstant(), DenseBase::fill()
+   */
+  /* This overload is needed because the usage of
+   *   using Base::operator=;
+   * fails on MSVC. Since the code below is working with GCC and MSVC, we skipped
+   * the usage of 'using'. This should be done only for operator=.
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array& operator=(const Scalar& value) {
+    Base::setConstant(value);
+    return *this;
+  }
+
+  /** Copies the value of the expression \a other into \c *this with automatic resizing.
+   *
+   * *this might be resized to match the dimensions of \a other. If *this was a null matrix (not already initialized),
+   * it will be initialized.
+   *
+   * Note that copying a row-vector into a vector (and conversely) is allowed.
+   * The resizing, if any, is then done in the appropriate way so that row-vectors
+   * remain row-vectors and vectors remain vectors.
+   */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array& operator=(const DenseBase<OtherDerived>& other) {
+    return Base::_set(other);
+  }
+
+  /**
+   * \brief Assigns arrays to each other.
+   *
+   * \note This is a special case of the templated operator=. Its purpose is
+   * to prevent a default operator= from hiding the templated operator=.
+   *
+   * \callgraph
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array& operator=(const Array& other) { return Base::_set(other); }
+
+  /** Default constructor.
+   *
+   * For fixed-size matrices, does nothing.
+   *
+   * For dynamic-size matrices, creates an empty matrix of size 0. Does not allocate any array. Such a matrix
+   * is called a null matrix. This constructor is the unique way to create null matrices: resizing
+   * a matrix to 0 is not supported.
+   *
+   * \sa resize(Index,Index)
+   */
+#ifdef EIGEN_INITIALIZE_COEFFS
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Array() : Base() { EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED }
+#else
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Array() = default;
+#endif
+  /** \brief Move constructor */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Array(Array&&) = default;
+  EIGEN_DEVICE_FUNC Array& operator=(Array&& other) EIGEN_NOEXCEPT_IF(std::is_nothrow_move_assignable<Scalar>::value) {
+    Base::operator=(std::move(other));
+    return *this;
+  }
+
+  /** \copydoc PlainObjectBase(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const
+   * ArgTypes&... args)
+   *
+   * Example: \include Array_variadic_ctor_cxx11.cpp
+   * Output: \verbinclude Array_variadic_ctor_cxx11.out
+   *
+   * \sa Array(const std::initializer_list<std::initializer_list<Scalar>>&)
+   * \sa Array(const Scalar&), Array(const Scalar&,const Scalar&)
+   */
+  template <typename... ArgTypes>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3,
+                                              const ArgTypes&... args)
+      : Base(a0, a1, a2, a3, args...) {}
+
+  /** \brief Constructs an array and initializes it from the coefficients given as initializer-lists grouped by row.
+   * \cpp11
+   *
+   * In the general case, the constructor takes a list of rows, each row being represented as a list of coefficients:
+   *
+   * Example: \include Array_initializer_list_23_cxx11.cpp
+   * Output: \verbinclude Array_initializer_list_23_cxx11.out
+   *
+   * Each of the inner initializer lists must contain the exact same number of elements, otherwise an assertion is
+   * triggered.
+   *
+   * In the case of a compile-time column 1D array, implicit transposition from a single row is allowed.
+   * Therefore <code> Array<int,Dynamic,1>{{1,2,3,4,5}}</code> is legal and the more verbose syntax
+   * <code>Array<int,Dynamic,1>{{1},{2},{3},{4},{5}}</code> can be avoided:
+   *
+   * Example: \include Array_initializer_list_vector_cxx11.cpp
+   * Output: \verbinclude Array_initializer_list_vector_cxx11.out
+   *
+   * In the case of fixed-sized arrays, the initializer list sizes must exactly match the array sizes,
+   * and implicit transposition is allowed for compile-time 1D arrays only.
+   *
+   * \sa  Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args)
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Array(
+      const std::initializer_list<std::initializer_list<Scalar>>& list)
+      : Base(list) {}
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  template <typename T>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Array(const T& x) {
+    Base::template _init1<T>(x);
+  }
+
+  template <typename T0, typename T1>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const T0& val0, const T1& val1) {
+    this->template _init2<T0, T1>(val0, val1);
+  }
+
+#else
+  /** \brief Constructs a fixed-sized array initialized with coefficients starting at \a data */
+  EIGEN_DEVICE_FUNC explicit Array(const Scalar* data);
+  /** Constructs a vector or row-vector with given dimension. \only_for_vectors
+   *
+   * Note that this is only useful for dynamic-size vectors. For fixed-size vectors,
+   * it is redundant to pass the dimension here, so it makes more sense to use the default
+   * constructor Array() instead.
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Array(Index dim);
+  /** constructs an initialized 1x1 Array with the given coefficient
+   * \sa const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args */
+  Array(const Scalar& value);
+  /** constructs an uninitialized array with \a rows rows and \a cols columns.
+   *
+   * This is useful for dynamic-size arrays. For fixed-size arrays,
+   * it is redundant to pass these parameters, so one should use the default constructor
+   * Array() instead. */
+  Array(Index rows, Index cols);
+  /** constructs an initialized 2D vector with given coefficients
+   * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args) */
+  Array(const Scalar& val0, const Scalar& val1);
+#endif  // end EIGEN_PARSED_BY_DOXYGEN
+
+  /** constructs an initialized 3D vector with given coefficients
+   * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args)
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Scalar& val0, const Scalar& val1, const Scalar& val2) {
+    EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Array, 3)
+    m_storage.data()[0] = val0;
+    m_storage.data()[1] = val1;
+    m_storage.data()[2] = val2;
+  }
+  /** constructs an initialized 4D vector with given coefficients
+   * \sa Array(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3, const ArgTypes&... args)
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(const Scalar& val0, const Scalar& val1, const Scalar& val2,
+                                              const Scalar& val3) {
+    EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Array, 4)
+    m_storage.data()[0] = val0;
+    m_storage.data()[1] = val1;
+    m_storage.data()[2] = val2;
+    m_storage.data()[3] = val3;
+  }
+
+  /** Copy constructor */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Array(const Array&) = default;
+
+ private:
+  struct PrivateType {};
+
+ public:
+  /** \sa MatrixBase::operator=(const EigenBase<OtherDerived>&) */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Array(
+      const EigenBase<OtherDerived>& other,
+      std::enable_if_t<internal::is_convertible<typename OtherDerived::Scalar, Scalar>::value, PrivateType> =
+          PrivateType())
+      : Base(other.derived()) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return 1; }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return this->innerSize(); }
+
+#ifdef EIGEN_ARRAY_PLUGIN
+#include EIGEN_ARRAY_PLUGIN
+#endif
+
+ private:
+  template <typename MatrixType, typename OtherDerived, bool SwapPointers>
+  friend struct internal::matrix_swap_impl;
+};
+
+/** \defgroup arraytypedefs Global array typedefs
+ * \ingroup Core_Module
+ *
+ * %Eigen defines several typedef shortcuts for most common 1D and 2D array types.
+ *
+ * The general patterns are the following:
+ *
+ * \c ArrayRowsColsType where \c Rows and \c Cols can be \c 2,\c 3,\c 4 for fixed size square matrices or \c X for
+ * dynamic size, and where \c Type can be \c i for integer, \c f for float, \c d for double, \c cf for complex float, \c
+ * cd for complex double.
+ *
+ * For example, \c Array33d is a fixed-size 3x3 array type of doubles, and \c ArrayXXf is a dynamic-size matrix of
+ * floats.
+ *
+ * There are also \c ArraySizeType which are self-explanatory. For example, \c Array4cf is
+ * a fixed-size 1D array of 4 complex floats.
+ *
+ * With \cpp11, template alias are also defined for common sizes.
+ * They follow the same pattern as above except that the scalar type suffix is replaced by a
+ * template parameter, i.e.:
+ *   - `ArrayRowsCols<Type>` where `Rows` and `Cols` can be \c 2,\c 3,\c 4, or \c X for fixed or dynamic size.
+ *   - `ArraySize<Type>` where `Size` can be \c 2,\c 3,\c 4 or \c X for fixed or dynamic size 1D arrays.
+ *
+ * \sa class Array
+ */
+
+#define EIGEN_MAKE_ARRAY_TYPEDEFS(Type, TypeSuffix, Size, SizeSuffix)        \
+  /** \ingroup arraytypedefs */                                              \
+  typedef Array<Type, Size, Size> Array##SizeSuffix##SizeSuffix##TypeSuffix; \
+  /** \ingroup arraytypedefs */                                              \
+  typedef Array<Type, Size, 1> Array##SizeSuffix##TypeSuffix;
+
+#define EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(Type, TypeSuffix, Size)  \
+  /** \ingroup arraytypedefs */                                  \
+  typedef Array<Type, Size, Dynamic> Array##Size##X##TypeSuffix; \
+  /** \ingroup arraytypedefs */                                  \
+  typedef Array<Type, Dynamic, Size> Array##X##Size##TypeSuffix;
+
+#define EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(Type, TypeSuffix) \
+  EIGEN_MAKE_ARRAY_TYPEDEFS(Type, TypeSuffix, 2, 2)           \
+  EIGEN_MAKE_ARRAY_TYPEDEFS(Type, TypeSuffix, 3, 3)           \
+  EIGEN_MAKE_ARRAY_TYPEDEFS(Type, TypeSuffix, 4, 4)           \
+  EIGEN_MAKE_ARRAY_TYPEDEFS(Type, TypeSuffix, Dynamic, X)     \
+  EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(Type, TypeSuffix, 2)        \
+  EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(Type, TypeSuffix, 3)        \
+  EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(Type, TypeSuffix, 4)
+
+EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(int, i)
+EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(float, f)
+EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(double, d)
+EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(std::complex<float>, cf)
+EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES(std::complex<double>, cd)
+
+#undef EIGEN_MAKE_ARRAY_TYPEDEFS_ALL_SIZES
+#undef EIGEN_MAKE_ARRAY_TYPEDEFS
+#undef EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS
+
+#define EIGEN_MAKE_ARRAY_TYPEDEFS(Size, SizeSuffix)              \
+  /** \ingroup arraytypedefs */                                  \
+  /** \brief \cpp11 */                                           \
+  template <typename Type>                                       \
+  using Array##SizeSuffix##SizeSuffix = Array<Type, Size, Size>; \
+  /** \ingroup arraytypedefs */                                  \
+  /** \brief \cpp11 */                                           \
+  template <typename Type>                                       \
+  using Array##SizeSuffix = Array<Type, Size, 1>;
+
+#define EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(Size)        \
+  /** \ingroup arraytypedefs */                      \
+  /** \brief \cpp11 */                               \
+  template <typename Type>                           \
+  using Array##Size##X = Array<Type, Size, Dynamic>; \
+  /** \ingroup arraytypedefs */                      \
+  /** \brief \cpp11 */                               \
+  template <typename Type>                           \
+  using Array##X##Size = Array<Type, Dynamic, Size>;
+
+EIGEN_MAKE_ARRAY_TYPEDEFS(2, 2)
+EIGEN_MAKE_ARRAY_TYPEDEFS(3, 3)
+EIGEN_MAKE_ARRAY_TYPEDEFS(4, 4)
+EIGEN_MAKE_ARRAY_TYPEDEFS(Dynamic, X)
+EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(2)
+EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(3)
+EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS(4)
+
+#undef EIGEN_MAKE_ARRAY_TYPEDEFS
+#undef EIGEN_MAKE_ARRAY_FIXED_TYPEDEFS
+
+#define EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, SizeSuffix) \
+  using Eigen::Matrix##SizeSuffix##TypeSuffix;                               \
+  using Eigen::Vector##SizeSuffix##TypeSuffix;                               \
+  using Eigen::RowVector##SizeSuffix##TypeSuffix;
+
+#define EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(TypeSuffix)       \
+  EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, 2) \
+  EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, 3) \
+  EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, 4) \
+  EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE_AND_SIZE(TypeSuffix, X)
+
+#define EIGEN_USING_ARRAY_TYPEDEFS        \
+  EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(i)  \
+  EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(f)  \
+  EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(d)  \
+  EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(cf) \
+  EIGEN_USING_ARRAY_TYPEDEFS_FOR_TYPE(cd)
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_ARRAY_H

include/Eigen/src/Core/ArrayBase.h

@@ -0,0 +1,222 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_ARRAYBASE_H
+#define EIGEN_ARRAYBASE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+template <typename ExpressionType>
+class MatrixWrapper;
+
+/** \class ArrayBase
+ * \ingroup Core_Module
+ *
+ * \brief Base class for all 1D and 2D array, and related expressions
+ *
+ * An array is similar to a dense vector or matrix. While matrices are mathematical
+ * objects with well defined linear algebra operators, an array is just a collection
+ * of scalar values arranged in a one or two dimensional fashion. As the main consequence,
+ * all operations applied to an array are performed coefficient wise. Furthermore,
+ * arrays support scalar math functions of the c++ standard library (e.g., std::sin(x)), and convenient
+ * constructors allowing to easily write generic code working for both scalar values
+ * and arrays.
+ *
+ * This class is the base that is inherited by all array expression types.
+ *
+ * \tparam Derived is the derived type, e.g., an array or an expression type.
+ *
+ * This class can be extended with the help of the plugin mechanism described on the page
+ * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_ARRAYBASE_PLUGIN.
+ *
+ * \sa class MatrixBase, \ref TopicClassHierarchy
+ */
+template <typename Derived>
+class ArrayBase : public DenseBase<Derived> {
+ public:
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** The base class for a given storage type. */
+  typedef ArrayBase StorageBaseType;
+
+  typedef ArrayBase Eigen_BaseClassForSpecializationOfGlobalMathFuncImpl;
+
+  typedef typename internal::traits<Derived>::StorageKind StorageKind;
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+  typedef typename internal::packet_traits<Scalar>::type PacketScalar;
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+
+  typedef DenseBase<Derived> Base;
+  using Base::ColsAtCompileTime;
+  using Base::Flags;
+  using Base::IsVectorAtCompileTime;
+  using Base::MaxColsAtCompileTime;
+  using Base::MaxRowsAtCompileTime;
+  using Base::MaxSizeAtCompileTime;
+  using Base::RowsAtCompileTime;
+  using Base::SizeAtCompileTime;
+
+  using Base::coeff;
+  using Base::coeffRef;
+  using Base::cols;
+  using Base::const_cast_derived;
+  using Base::derived;
+  using Base::lazyAssign;
+  using Base::rows;
+  using Base::size;
+  using Base::operator-;
+  using Base::operator=;
+  using Base::operator+=;
+  using Base::operator-=;
+  using Base::operator*=;
+  using Base::operator/=;
+
+  typedef typename Base::CoeffReturnType CoeffReturnType;
+
+#endif  // not EIGEN_PARSED_BY_DOXYGEN
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef typename Base::PlainObject PlainObject;
+
+  /** \internal Represents a matrix with all coefficients equal to one another*/
+  typedef CwiseNullaryOp<internal::scalar_constant_op<Scalar>, PlainObject> ConstantReturnType;
+#endif  // not EIGEN_PARSED_BY_DOXYGEN
+
+#define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::ArrayBase
+#define EIGEN_DOC_UNARY_ADDONS(X, Y)
+#include "../plugins/MatrixCwiseUnaryOps.inc"
+#include "../plugins/ArrayCwiseUnaryOps.inc"
+#include "../plugins/CommonCwiseBinaryOps.inc"
+#include "../plugins/MatrixCwiseBinaryOps.inc"
+#include "../plugins/ArrayCwiseBinaryOps.inc"
+#ifdef EIGEN_ARRAYBASE_PLUGIN
+#include EIGEN_ARRAYBASE_PLUGIN
+#endif
+#undef EIGEN_CURRENT_STORAGE_BASE_CLASS
+#undef EIGEN_DOC_UNARY_ADDONS
+
+  /** Special case of the template operator=, in order to prevent the compiler
+   * from generating a default operator= (issue hit with g++ 4.1)
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const ArrayBase& other) {
+    internal::call_assignment(derived(), other.derived());
+    return derived();
+  }
+
+  /** Set all the entries to \a value.
+   * \sa DenseBase::setConstant(), DenseBase::fill() */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const Scalar& value) {
+    Base::setConstant(value);
+    return derived();
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator+=(const Scalar& scalar);
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator-=(const Scalar& scalar);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator+=(const ArrayBase<OtherDerived>& other);
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator-=(const ArrayBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator*=(const ArrayBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator/=(const ArrayBase<OtherDerived>& other);
+
+ public:
+  EIGEN_DEVICE_FUNC ArrayBase<Derived>& array() { return *this; }
+  EIGEN_DEVICE_FUNC const ArrayBase<Derived>& array() const { return *this; }
+
+  /** \returns an \link Eigen::MatrixBase Matrix \endlink expression of this array
+   * \sa MatrixBase::array() */
+  EIGEN_DEVICE_FUNC MatrixWrapper<Derived> matrix() { return MatrixWrapper<Derived>(derived()); }
+  EIGEN_DEVICE_FUNC const MatrixWrapper<const Derived> matrix() const {
+    return MatrixWrapper<const Derived>(derived());
+  }
+
+  //     template<typename Dest>
+  //     inline void evalTo(Dest& dst) const { dst = matrix(); }
+
+ protected:
+  EIGEN_DEFAULT_COPY_CONSTRUCTOR(ArrayBase)
+  EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(ArrayBase)
+
+ private:
+  explicit ArrayBase(Index);
+  ArrayBase(Index, Index);
+  template <typename OtherDerived>
+  explicit ArrayBase(const ArrayBase<OtherDerived>&);
+
+ protected:
+  // mixing arrays and matrices is not legal
+  template <typename OtherDerived>
+  Derived& operator+=(const MatrixBase<OtherDerived>&) {
+    EIGEN_STATIC_ASSERT(std::ptrdiff_t(sizeof(typename OtherDerived::Scalar)) == -1,
+                        YOU_CANNOT_MIX_ARRAYS_AND_MATRICES);
+    return *this;
+  }
+  // mixing arrays and matrices is not legal
+  template <typename OtherDerived>
+  Derived& operator-=(const MatrixBase<OtherDerived>&) {
+    EIGEN_STATIC_ASSERT(std::ptrdiff_t(sizeof(typename OtherDerived::Scalar)) == -1,
+                        YOU_CANNOT_MIX_ARRAYS_AND_MATRICES);
+    return *this;
+  }
+};
+
+/** replaces \c *this by \c *this - \a other.
+ *
+ * \returns a reference to \c *this
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase<Derived>::operator-=(const ArrayBase<OtherDerived>& other) {
+  call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar, typename OtherDerived::Scalar>());
+  return derived();
+}
+
+/** replaces \c *this by \c *this + \a other.
+ *
+ * \returns a reference to \c *this
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase<Derived>::operator+=(const ArrayBase<OtherDerived>& other) {
+  call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar, typename OtherDerived::Scalar>());
+  return derived();
+}
+
+/** replaces \c *this by \c *this * \a other coefficient wise.
+ *
+ * \returns a reference to \c *this
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase<Derived>::operator*=(const ArrayBase<OtherDerived>& other) {
+  call_assignment(derived(), other.derived(), internal::mul_assign_op<Scalar, typename OtherDerived::Scalar>());
+  return derived();
+}
+
+/** replaces \c *this by \c *this / \a other coefficient wise.
+ *
+ * \returns a reference to \c *this
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase<Derived>::operator/=(const ArrayBase<OtherDerived>& other) {
+  call_assignment(derived(), other.derived(), internal::div_assign_op<Scalar, typename OtherDerived::Scalar>());
+  return derived();
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_ARRAYBASE_H

include/Eigen/src/Core/ArrayWrapper.h

@@ -0,0 +1,173 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_ARRAYWRAPPER_H
+#define EIGEN_ARRAYWRAPPER_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class ArrayWrapper
+ * \ingroup Core_Module
+ *
+ * \brief Expression of a mathematical vector or matrix as an array object
+ *
+ * This class is the return type of MatrixBase::array(), and most of the time
+ * this is the only way it is use.
+ *
+ * \sa MatrixBase::array(), class MatrixWrapper
+ */
+
+namespace internal {
+template <typename ExpressionType>
+struct traits<ArrayWrapper<ExpressionType> > : public traits<remove_all_t<typename ExpressionType::Nested> > {
+  typedef ArrayXpr XprKind;
+  // Let's remove NestByRefBit
+  enum {
+    Flags0 = traits<remove_all_t<typename ExpressionType::Nested> >::Flags,
+    LvalueBitFlag = is_lvalue<ExpressionType>::value ? LvalueBit : 0,
+    Flags = (Flags0 & ~(NestByRefBit | LvalueBit)) | LvalueBitFlag
+  };
+};
+}  // namespace internal
+
+template <typename ExpressionType>
+class ArrayWrapper : public ArrayBase<ArrayWrapper<ExpressionType> > {
+ public:
+  typedef ArrayBase<ArrayWrapper> Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(ArrayWrapper)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ArrayWrapper)
+  typedef internal::remove_all_t<ExpressionType> NestedExpression;
+
+  typedef std::conditional_t<internal::is_lvalue<ExpressionType>::value, Scalar, const Scalar>
+      ScalarWithConstIfNotLvalue;
+
+  typedef typename internal::ref_selector<ExpressionType>::non_const_type NestedExpressionType;
+
+  using Base::coeffRef;
+
+  EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE ArrayWrapper(ExpressionType& matrix) : m_expression(matrix) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_expression.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_expression.cols(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT {
+    return m_expression.outerStride();
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT {
+    return m_expression.innerStride();
+  }
+
+  EIGEN_DEVICE_FUNC constexpr ScalarWithConstIfNotLvalue* data() { return m_expression.data(); }
+  EIGEN_DEVICE_FUNC constexpr const Scalar* data() const { return m_expression.data(); }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const {
+    return m_expression.coeffRef(rowId, colId);
+  }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const { return m_expression.coeffRef(index); }
+
+  template <typename Dest>
+  EIGEN_DEVICE_FUNC inline void evalTo(Dest& dst) const {
+    dst = m_expression;
+  }
+
+  EIGEN_DEVICE_FUNC const internal::remove_all_t<NestedExpressionType>& nestedExpression() const {
+    return m_expression;
+  }
+
+  /** Forwards the resizing request to the nested expression
+   * \sa DenseBase::resize(Index)  */
+  EIGEN_DEVICE_FUNC void resize(Index newSize) { m_expression.resize(newSize); }
+  /** Forwards the resizing request to the nested expression
+   * \sa DenseBase::resize(Index,Index)*/
+  EIGEN_DEVICE_FUNC void resize(Index rows, Index cols) { m_expression.resize(rows, cols); }
+
+ protected:
+  NestedExpressionType m_expression;
+};
+
+/** \class MatrixWrapper
+ * \ingroup Core_Module
+ *
+ * \brief Expression of an array as a mathematical vector or matrix
+ *
+ * This class is the return type of ArrayBase::matrix(), and most of the time
+ * this is the only way it is use.
+ *
+ * \sa MatrixBase::matrix(), class ArrayWrapper
+ */
+
+namespace internal {
+template <typename ExpressionType>
+struct traits<MatrixWrapper<ExpressionType> > : public traits<remove_all_t<typename ExpressionType::Nested> > {
+  typedef MatrixXpr XprKind;
+  // Let's remove NestByRefBit
+  enum {
+    Flags0 = traits<remove_all_t<typename ExpressionType::Nested> >::Flags,
+    LvalueBitFlag = is_lvalue<ExpressionType>::value ? LvalueBit : 0,
+    Flags = (Flags0 & ~(NestByRefBit | LvalueBit)) | LvalueBitFlag
+  };
+};
+}  // namespace internal
+
+template <typename ExpressionType>
+class MatrixWrapper : public MatrixBase<MatrixWrapper<ExpressionType> > {
+ public:
+  typedef MatrixBase<MatrixWrapper<ExpressionType> > Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(MatrixWrapper)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(MatrixWrapper)
+  typedef internal::remove_all_t<ExpressionType> NestedExpression;
+
+  typedef std::conditional_t<internal::is_lvalue<ExpressionType>::value, Scalar, const Scalar>
+      ScalarWithConstIfNotLvalue;
+
+  typedef typename internal::ref_selector<ExpressionType>::non_const_type NestedExpressionType;
+
+  using Base::coeffRef;
+
+  EIGEN_DEVICE_FUNC explicit inline MatrixWrapper(ExpressionType& matrix) : m_expression(matrix) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_expression.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_expression.cols(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT {
+    return m_expression.outerStride();
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT {
+    return m_expression.innerStride();
+  }
+
+  EIGEN_DEVICE_FUNC constexpr ScalarWithConstIfNotLvalue* data() { return m_expression.data(); }
+  EIGEN_DEVICE_FUNC constexpr const Scalar* data() const { return m_expression.data(); }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const {
+    return m_expression.derived().coeffRef(rowId, colId);
+  }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const { return m_expression.coeffRef(index); }
+
+  EIGEN_DEVICE_FUNC const internal::remove_all_t<NestedExpressionType>& nestedExpression() const {
+    return m_expression;
+  }
+
+  /** Forwards the resizing request to the nested expression
+   * \sa DenseBase::resize(Index)  */
+  EIGEN_DEVICE_FUNC void resize(Index newSize) { m_expression.resize(newSize); }
+  /** Forwards the resizing request to the nested expression
+   * \sa DenseBase::resize(Index,Index)*/
+  EIGEN_DEVICE_FUNC void resize(Index rows, Index cols) { m_expression.resize(rows, cols); }
+
+ protected:
+  NestedExpressionType m_expression;
+};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_ARRAYWRAPPER_H

include/Eigen/src/Core/Assign.h

@@ -0,0 +1,80 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2007 Michael Olbrich <michael.olbrich@gmx.net>
+// Copyright (C) 2006-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_ASSIGN_H
+#define EIGEN_ASSIGN_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::lazyAssign(const DenseBase<OtherDerived>& other) {
+  enum { SameType = internal::is_same<typename Derived::Scalar, typename OtherDerived::Scalar>::value };
+
+  EIGEN_STATIC_ASSERT_LVALUE(Derived)
+  EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Derived, OtherDerived)
+  EIGEN_STATIC_ASSERT(
+      SameType,
+      YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY)
+
+  eigen_assert(rows() == other.rows() && cols() == other.cols());
+  internal::call_assignment_no_alias(derived(), other.derived());
+
+  return derived();
+}
+
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator=(const DenseBase<OtherDerived>& other) {
+  internal::call_assignment(derived(), other.derived());
+  return derived();
+}
+
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator=(const DenseBase& other) {
+  internal::call_assignment(derived(), other.derived());
+  return derived();
+}
+
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const MatrixBase& other) {
+  internal::call_assignment(derived(), other.derived());
+  return derived();
+}
+
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const DenseBase<OtherDerived>& other) {
+  internal::call_assignment(derived(), other.derived());
+  return derived();
+}
+
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const EigenBase<OtherDerived>& other) {
+  internal::call_assignment(derived(), other.derived());
+  return derived();
+}
+
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(
+    const ReturnByValue<OtherDerived>& other) {
+  other.derived().evalTo(derived());
+  return derived();
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_ASSIGN_H

include/Eigen/src/Core/AssignEvaluator.h

@@ -0,0 +1,965 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2011 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2011-2014 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2011-2012 Jitse Niesen <jitse@maths.leeds.ac.uk>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_ASSIGN_EVALUATOR_H
+#define EIGEN_ASSIGN_EVALUATOR_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+// This implementation is based on Assign.h
+
+namespace internal {
+
+/***************************************************************************
+ * Part 1 : the logic deciding a strategy for traversal and unrolling       *
+ ***************************************************************************/
+
+// copy_using_evaluator_traits is based on assign_traits
+
+template <typename DstEvaluator, typename SrcEvaluator, typename AssignFunc, int MaxPacketSize = -1>
+struct copy_using_evaluator_traits {
+  typedef typename DstEvaluator::XprType Dst;
+  typedef typename Dst::Scalar DstScalar;
+
+  enum { DstFlags = DstEvaluator::Flags, SrcFlags = SrcEvaluator::Flags };
+
+ public:
+  enum {
+    DstAlignment = DstEvaluator::Alignment,
+    SrcAlignment = SrcEvaluator::Alignment,
+    DstHasDirectAccess = (DstFlags & DirectAccessBit) == DirectAccessBit,
+    JointAlignment = plain_enum_min(DstAlignment, SrcAlignment)
+  };
+
+ private:
+  enum {
+    InnerSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::SizeAtCompileTime)
+                : int(DstFlags) & RowMajorBit   ? int(Dst::ColsAtCompileTime)
+                                                : int(Dst::RowsAtCompileTime),
+    InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime)
+                   : int(DstFlags) & RowMajorBit   ? int(Dst::MaxColsAtCompileTime)
+                                                   : int(Dst::MaxRowsAtCompileTime),
+    RestrictedInnerSize = min_size_prefer_fixed(InnerSize, MaxPacketSize),
+    RestrictedLinearSize = min_size_prefer_fixed(Dst::SizeAtCompileTime, MaxPacketSize),
+    OuterStride = int(outer_stride_at_compile_time<Dst>::ret),
+    MaxSizeAtCompileTime = Dst::SizeAtCompileTime
+  };
+
+  // TODO distinguish between linear traversal and inner-traversals
+  typedef typename find_best_packet<DstScalar, RestrictedLinearSize>::type LinearPacketType;
+  typedef typename find_best_packet<DstScalar, RestrictedInnerSize>::type InnerPacketType;
+
+  enum {
+    LinearPacketSize = unpacket_traits<LinearPacketType>::size,
+    InnerPacketSize = unpacket_traits<InnerPacketType>::size
+  };
+
+ public:
+  enum {
+    LinearRequiredAlignment = unpacket_traits<LinearPacketType>::alignment,
+    InnerRequiredAlignment = unpacket_traits<InnerPacketType>::alignment
+  };
+
+ private:
+  enum {
+    DstIsRowMajor = DstFlags & RowMajorBit,
+    SrcIsRowMajor = SrcFlags & RowMajorBit,
+    StorageOrdersAgree = (int(DstIsRowMajor) == int(SrcIsRowMajor)),
+    MightVectorize = bool(StorageOrdersAgree) && (int(DstFlags) & int(SrcFlags) & ActualPacketAccessBit) &&
+                     bool(functor_traits<AssignFunc>::PacketAccess),
+    MayInnerVectorize = MightVectorize && int(InnerSize) != Dynamic && int(InnerSize) % int(InnerPacketSize) == 0 &&
+                        int(OuterStride) != Dynamic && int(OuterStride) % int(InnerPacketSize) == 0 &&
+                        (EIGEN_UNALIGNED_VECTORIZE || int(JointAlignment) >= int(InnerRequiredAlignment)),
+    MayLinearize = bool(StorageOrdersAgree) && (int(DstFlags) & int(SrcFlags) & LinearAccessBit),
+    MayLinearVectorize = bool(MightVectorize) && bool(MayLinearize) && bool(DstHasDirectAccess) &&
+                         (EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment) >= int(LinearRequiredAlignment)) ||
+                          MaxSizeAtCompileTime == Dynamic),
+    /* If the destination isn't aligned, we have to do runtime checks and we don't unroll,
+       so it's only good for large enough sizes. */
+    MaySliceVectorize = bool(MightVectorize) && bool(DstHasDirectAccess) &&
+                        (int(InnerMaxSize) == Dynamic ||
+                         int(InnerMaxSize) >= (EIGEN_UNALIGNED_VECTORIZE ? InnerPacketSize : (3 * InnerPacketSize)))
+    /* slice vectorization can be slow, so we only want it if the slices are big, which is
+       indicated by InnerMaxSize rather than InnerSize, think of the case of a dynamic block
+       in a fixed-size matrix
+       However, with EIGEN_UNALIGNED_VECTORIZE and unrolling, slice vectorization is still worth it */
+  };
+
+ public:
+  enum {
+    Traversal = int(Dst::SizeAtCompileTime) == 0
+                    ? int(AllAtOnceTraversal)  // If compile-size is zero, traversing will fail at compile-time.
+                : (int(MayLinearVectorize) && (LinearPacketSize > InnerPacketSize)) ? int(LinearVectorizedTraversal)
+                : int(MayInnerVectorize)                                            ? int(InnerVectorizedTraversal)
+                : int(MayLinearVectorize)                                           ? int(LinearVectorizedTraversal)
+                : int(MaySliceVectorize)                                            ? int(SliceVectorizedTraversal)
+                : int(MayLinearize)                                                 ? int(LinearTraversal)
+                                                                                    : int(DefaultTraversal),
+    Vectorized = int(Traversal) == InnerVectorizedTraversal || int(Traversal) == LinearVectorizedTraversal ||
+                 int(Traversal) == SliceVectorizedTraversal
+  };
+
+  typedef std::conditional_t<int(Traversal) == LinearVectorizedTraversal, LinearPacketType, InnerPacketType> PacketType;
+
+ private:
+  enum {
+    ActualPacketSize = int(Traversal) == LinearVectorizedTraversal ? LinearPacketSize
+                       : Vectorized                                ? InnerPacketSize
+                                                                   : 1,
+    UnrollingLimit = EIGEN_UNROLLING_LIMIT * ActualPacketSize,
+    MayUnrollCompletely =
+        int(Dst::SizeAtCompileTime) != Dynamic &&
+        int(Dst::SizeAtCompileTime) * (int(DstEvaluator::CoeffReadCost) + int(SrcEvaluator::CoeffReadCost)) <=
+            int(UnrollingLimit),
+    MayUnrollInner =
+        int(InnerSize) != Dynamic &&
+        int(InnerSize) * (int(DstEvaluator::CoeffReadCost) + int(SrcEvaluator::CoeffReadCost)) <= int(UnrollingLimit)
+  };
+
+ public:
+  enum {
+    Unrolling = (int(Traversal) == int(InnerVectorizedTraversal) || int(Traversal) == int(DefaultTraversal))
+                    ? (int(MayUnrollCompletely) ? int(CompleteUnrolling)
+                       : int(MayUnrollInner)    ? int(InnerUnrolling)
+                                                : int(NoUnrolling))
+                : int(Traversal) == int(LinearVectorizedTraversal)
+                    ? (bool(MayUnrollCompletely) &&
+                               (EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment) >= int(LinearRequiredAlignment)))
+                           ? int(CompleteUnrolling)
+                           : int(NoUnrolling))
+                : int(Traversal) == int(LinearTraversal)
+                    ? (bool(MayUnrollCompletely) ? int(CompleteUnrolling) : int(NoUnrolling))
+#if EIGEN_UNALIGNED_VECTORIZE
+                : int(Traversal) == int(SliceVectorizedTraversal)
+                    ? (bool(MayUnrollInner) ? int(InnerUnrolling) : int(NoUnrolling))
+#endif
+                    : int(NoUnrolling)
+  };
+
+#ifdef EIGEN_DEBUG_ASSIGN
+  static void debug() {
+    std::cerr << "DstXpr: " << typeid(typename DstEvaluator::XprType).name() << std::endl;
+    std::cerr << "SrcXpr: " << typeid(typename SrcEvaluator::XprType).name() << std::endl;
+    std::cerr.setf(std::ios::hex, std::ios::basefield);
+    std::cerr << "DstFlags"
+              << " = " << DstFlags << " (" << demangle_flags(DstFlags) << " )" << std::endl;
+    std::cerr << "SrcFlags"
+              << " = " << SrcFlags << " (" << demangle_flags(SrcFlags) << " )" << std::endl;
+    std::cerr.unsetf(std::ios::hex);
+    EIGEN_DEBUG_VAR(DstAlignment)
+    EIGEN_DEBUG_VAR(SrcAlignment)
+    EIGEN_DEBUG_VAR(LinearRequiredAlignment)
+    EIGEN_DEBUG_VAR(InnerRequiredAlignment)
+    EIGEN_DEBUG_VAR(JointAlignment)
+    EIGEN_DEBUG_VAR(InnerSize)
+    EIGEN_DEBUG_VAR(InnerMaxSize)
+    EIGEN_DEBUG_VAR(LinearPacketSize)
+    EIGEN_DEBUG_VAR(InnerPacketSize)
+    EIGEN_DEBUG_VAR(ActualPacketSize)
+    EIGEN_DEBUG_VAR(StorageOrdersAgree)
+    EIGEN_DEBUG_VAR(MightVectorize)
+    EIGEN_DEBUG_VAR(MayLinearize)
+    EIGEN_DEBUG_VAR(MayInnerVectorize)
+    EIGEN_DEBUG_VAR(MayLinearVectorize)
+    EIGEN_DEBUG_VAR(MaySliceVectorize)
+    std::cerr << "Traversal"
+              << " = " << Traversal << " (" << demangle_traversal(Traversal) << ")" << std::endl;
+    EIGEN_DEBUG_VAR(SrcEvaluator::CoeffReadCost)
+    EIGEN_DEBUG_VAR(DstEvaluator::CoeffReadCost)
+    EIGEN_DEBUG_VAR(Dst::SizeAtCompileTime)
+    EIGEN_DEBUG_VAR(UnrollingLimit)
+    EIGEN_DEBUG_VAR(MayUnrollCompletely)
+    EIGEN_DEBUG_VAR(MayUnrollInner)
+    std::cerr << "Unrolling"
+              << " = " << Unrolling << " (" << demangle_unrolling(Unrolling) << ")" << std::endl;
+    std::cerr << std::endl;
+  }
+#endif
+};
+
+/***************************************************************************
+ * Part 2 : meta-unrollers
+ ***************************************************************************/
+
+/************************
+*** Default traversal ***
+************************/
+
+template <typename Kernel, int Index, int Stop>
+struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling {
+  // FIXME: this is not very clean, perhaps this information should be provided by the kernel?
+  typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
+  typedef typename DstEvaluatorType::XprType DstXprType;
+
+  enum { outer = Index / DstXprType::InnerSizeAtCompileTime, inner = Index % DstXprType::InnerSizeAtCompileTime };
+
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
+    kernel.assignCoeffByOuterInner(outer, inner);
+    copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Index + 1, Stop>::run(kernel);
+  }
+};
+
+template <typename Kernel, int Stop>
+struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Stop, Stop> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel&) {}
+};
+
+template <typename Kernel, int Index_, int Stop>
+struct copy_using_evaluator_DefaultTraversal_InnerUnrolling {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel, Index outer) {
+    kernel.assignCoeffByOuterInner(outer, Index_);
+    copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Index_ + 1, Stop>::run(kernel, outer);
+  }
+};
+
+template <typename Kernel, int Stop>
+struct copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Stop, Stop> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&, Index) {}
+};
+
+/***********************
+*** Linear traversal ***
+***********************/
+
+template <typename Kernel, int Index, int Stop>
+struct copy_using_evaluator_LinearTraversal_CompleteUnrolling {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
+    kernel.assignCoeff(Index);
+    copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Index + 1, Stop>::run(kernel);
+  }
+};
+
+template <typename Kernel, int Stop>
+struct copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Stop, Stop> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) {}
+};
+
+/**************************
+*** Inner vectorization ***
+**************************/
+
+template <typename Kernel, int Index, int Stop>
+struct copy_using_evaluator_innervec_CompleteUnrolling {
+  // FIXME: this is not very clean, perhaps this information should be provided by the kernel?
+  typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
+  typedef typename DstEvaluatorType::XprType DstXprType;
+  typedef typename Kernel::PacketType PacketType;
+
+  enum {
+    outer = Index / DstXprType::InnerSizeAtCompileTime,
+    inner = Index % DstXprType::InnerSizeAtCompileTime,
+    SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
+    DstAlignment = Kernel::AssignmentTraits::DstAlignment
+  };
+
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
+    kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
+    enum { NextIndex = Index + unpacket_traits<PacketType>::size };
+    copy_using_evaluator_innervec_CompleteUnrolling<Kernel, NextIndex, Stop>::run(kernel);
+  }
+};
+
+template <typename Kernel, int Stop>
+struct copy_using_evaluator_innervec_CompleteUnrolling<Kernel, Stop, Stop> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel&) {}
+};
+
+template <typename Kernel, int Index_, int Stop, int SrcAlignment, int DstAlignment>
+struct copy_using_evaluator_innervec_InnerUnrolling {
+  typedef typename Kernel::PacketType PacketType;
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel, Index outer) {
+    kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, Index_);
+    enum { NextIndex = Index_ + unpacket_traits<PacketType>::size };
+    copy_using_evaluator_innervec_InnerUnrolling<Kernel, NextIndex, Stop, SrcAlignment, DstAlignment>::run(kernel,
+                                                                                                           outer);
+  }
+};
+
+template <typename Kernel, int Stop, int SrcAlignment, int DstAlignment>
+struct copy_using_evaluator_innervec_InnerUnrolling<Kernel, Stop, Stop, SrcAlignment, DstAlignment> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&, Index) {}
+};
+
+/***************************************************************************
+ * Part 3 : implementation of all cases
+ ***************************************************************************/
+
+// dense_assignment_loop is based on assign_impl
+
+template <typename Kernel, int Traversal = Kernel::AssignmentTraits::Traversal,
+          int Unrolling = Kernel::AssignmentTraits::Unrolling>
+struct dense_assignment_loop;
+
+/************************
+***** Special Cases *****
+************************/
+
+// Zero-sized assignment is a no-op.
+template <typename Kernel, int Unrolling>
+struct dense_assignment_loop<Kernel, AllAtOnceTraversal, Unrolling> {
+  EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE EIGEN_CONSTEXPR run(Kernel& /*kernel*/) {
+    EIGEN_STATIC_ASSERT(int(Kernel::DstEvaluatorType::XprType::SizeAtCompileTime) == 0,
+                        EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT)
+  }
+};
+
+/************************
+*** Default traversal ***
+************************/
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, DefaultTraversal, NoUnrolling> {
+  EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE run(Kernel& kernel) {
+    for (Index outer = 0; outer < kernel.outerSize(); ++outer) {
+      for (Index inner = 0; inner < kernel.innerSize(); ++inner) {
+        kernel.assignCoeffByOuterInner(outer, inner);
+      }
+    }
+  }
+};
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, DefaultTraversal, CompleteUnrolling> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
+    typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
+    copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
+  }
+};
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, DefaultTraversal, InnerUnrolling> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
+    typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
+
+    const Index outerSize = kernel.outerSize();
+    for (Index outer = 0; outer < outerSize; ++outer)
+      copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime>::run(kernel,
+                                                                                                               outer);
+  }
+};
+
+/***************************
+*** Linear vectorization ***
+***************************/
+
+// The goal of unaligned_dense_assignment_loop is simply to factorize the handling
+// of the non vectorizable beginning and ending parts
+
+template <bool IsAligned = false>
+struct unaligned_dense_assignment_loop {
+  // if IsAligned = true, then do nothing
+  template <typename Kernel>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel&, Index, Index) {}
+};
+
+template <>
+struct unaligned_dense_assignment_loop<false> {
+  // MSVC must not inline this functions. If it does, it fails to optimize the
+  // packet access path.
+  // FIXME check which version exhibits this issue
+#if EIGEN_COMP_MSVC
+  template <typename Kernel>
+  static EIGEN_DONT_INLINE void run(Kernel& kernel, Index start, Index end)
+#else
+  template <typename Kernel>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel, Index start, Index end)
+#endif
+  {
+    for (Index index = start; index < end; ++index) kernel.assignCoeff(index);
+  }
+};
+
+template <typename Kernel, int Index, int Stop>
+struct copy_using_evaluator_linearvec_CompleteUnrolling {
+  // FIXME: this is not very clean, perhaps this information should be provided by the kernel?
+  typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
+  typedef typename DstEvaluatorType::XprType DstXprType;
+  typedef typename Kernel::PacketType PacketType;
+
+  enum { SrcAlignment = Kernel::AssignmentTraits::SrcAlignment, DstAlignment = Kernel::AssignmentTraits::DstAlignment };
+
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
+    kernel.template assignPacket<DstAlignment, SrcAlignment, PacketType>(Index);
+    enum { NextIndex = Index + unpacket_traits<PacketType>::size };
+    copy_using_evaluator_linearvec_CompleteUnrolling<Kernel, NextIndex, Stop>::run(kernel);
+  }
+};
+
+template <typename Kernel, int Stop>
+struct copy_using_evaluator_linearvec_CompleteUnrolling<Kernel, Stop, Stop> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel&) {}
+};
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, NoUnrolling> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel) {
+    const Index size = kernel.size();
+    typedef typename Kernel::Scalar Scalar;
+    typedef typename Kernel::PacketType PacketType;
+    enum {
+      requestedAlignment = Kernel::AssignmentTraits::LinearRequiredAlignment,
+      packetSize = unpacket_traits<PacketType>::size,
+      dstIsAligned = int(Kernel::AssignmentTraits::DstAlignment) >= int(requestedAlignment),
+      dstAlignment = packet_traits<Scalar>::AlignedOnScalar ? int(requestedAlignment)
+                                                            : int(Kernel::AssignmentTraits::DstAlignment),
+      srcAlignment = Kernel::AssignmentTraits::JointAlignment
+    };
+    const Index alignedStart =
+        dstIsAligned ? 0 : internal::first_aligned<requestedAlignment>(kernel.dstDataPtr(), size);
+    const Index alignedEnd = alignedStart + ((size - alignedStart) / packetSize) * packetSize;
+
+    unaligned_dense_assignment_loop<dstIsAligned != 0>::run(kernel, 0, alignedStart);
+
+    for (Index index = alignedStart; index < alignedEnd; index += packetSize)
+      kernel.template assignPacket<dstAlignment, srcAlignment, PacketType>(index);
+
+    unaligned_dense_assignment_loop<>::run(kernel, alignedEnd, size);
+  }
+};
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, CompleteUnrolling> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel) {
+    typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
+    typedef typename Kernel::PacketType PacketType;
+
+    enum {
+      size = DstXprType::SizeAtCompileTime,
+      packetSize = unpacket_traits<PacketType>::size,
+      alignedSize = (int(size) / packetSize) * packetSize
+    };
+
+    copy_using_evaluator_linearvec_CompleteUnrolling<Kernel, 0, alignedSize>::run(kernel);
+    copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, alignedSize, size>::run(kernel);
+  }
+};
+
+/**************************
+*** Inner vectorization ***
+**************************/
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, NoUnrolling> {
+  typedef typename Kernel::PacketType PacketType;
+  enum { SrcAlignment = Kernel::AssignmentTraits::SrcAlignment, DstAlignment = Kernel::AssignmentTraits::DstAlignment };
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel) {
+    const Index innerSize = kernel.innerSize();
+    const Index outerSize = kernel.outerSize();
+    const Index packetSize = unpacket_traits<PacketType>::size;
+    for (Index outer = 0; outer < outerSize; ++outer)
+      for (Index inner = 0; inner < innerSize; inner += packetSize)
+        kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
+  }
+};
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, CompleteUnrolling> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
+    typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
+    copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
+  }
+};
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, InnerUnrolling> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel) {
+    typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
+    typedef typename Kernel::AssignmentTraits Traits;
+    const Index outerSize = kernel.outerSize();
+    for (Index outer = 0; outer < outerSize; ++outer)
+      copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime, Traits::SrcAlignment,
+                                                   Traits::DstAlignment>::run(kernel, outer);
+  }
+};
+
+/***********************
+*** Linear traversal ***
+***********************/
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, LinearTraversal, NoUnrolling> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel) {
+    const Index size = kernel.size();
+    for (Index i = 0; i < size; ++i) kernel.assignCoeff(i);
+  }
+};
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, LinearTraversal, CompleteUnrolling> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel) {
+    typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
+    copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
+  }
+};
+
+/**************************
+*** Slice vectorization ***
+***************************/
+
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, NoUnrolling> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel) {
+    typedef typename Kernel::Scalar Scalar;
+    typedef typename Kernel::PacketType PacketType;
+    enum {
+      packetSize = unpacket_traits<PacketType>::size,
+      requestedAlignment = int(Kernel::AssignmentTraits::InnerRequiredAlignment),
+      alignable =
+          packet_traits<Scalar>::AlignedOnScalar || int(Kernel::AssignmentTraits::DstAlignment) >= sizeof(Scalar),
+      dstIsAligned = int(Kernel::AssignmentTraits::DstAlignment) >= int(requestedAlignment),
+      dstAlignment = alignable ? int(requestedAlignment) : int(Kernel::AssignmentTraits::DstAlignment)
+    };
+    const Scalar* dst_ptr = kernel.dstDataPtr();
+    if ((!bool(dstIsAligned)) && (std::uintptr_t(dst_ptr) % sizeof(Scalar)) > 0) {
+      // the pointer is not aligned-on scalar, so alignment is not possible
+      return dense_assignment_loop<Kernel, DefaultTraversal, NoUnrolling>::run(kernel);
+    }
+    const Index packetAlignedMask = packetSize - 1;
+    const Index innerSize = kernel.innerSize();
+    const Index outerSize = kernel.outerSize();
+    const Index alignedStep = alignable ? (packetSize - kernel.outerStride() % packetSize) & packetAlignedMask : 0;
+    Index alignedStart =
+        ((!alignable) || bool(dstIsAligned)) ? 0 : internal::first_aligned<requestedAlignment>(dst_ptr, innerSize);
+
+    for (Index outer = 0; outer < outerSize; ++outer) {
+      const Index alignedEnd = alignedStart + ((innerSize - alignedStart) & ~packetAlignedMask);
+      // do the non-vectorizable part of the assignment
+      for (Index inner = 0; inner < alignedStart; ++inner) kernel.assignCoeffByOuterInner(outer, inner);
+
+      // do the vectorizable part of the assignment
+      for (Index inner = alignedStart; inner < alignedEnd; inner += packetSize)
+        kernel.template assignPacketByOuterInner<dstAlignment, Unaligned, PacketType>(outer, inner);
+
+      // do the non-vectorizable part of the assignment
+      for (Index inner = alignedEnd; inner < innerSize; ++inner) kernel.assignCoeffByOuterInner(outer, inner);
+
+      alignedStart = numext::mini((alignedStart + alignedStep) % packetSize, innerSize);
+    }
+  }
+};
+
+#if EIGEN_UNALIGNED_VECTORIZE
+template <typename Kernel>
+struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, InnerUnrolling> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel) {
+    typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
+    typedef typename Kernel::PacketType PacketType;
+
+    enum {
+      innerSize = DstXprType::InnerSizeAtCompileTime,
+      packetSize = unpacket_traits<PacketType>::size,
+      vectorizableSize = (int(innerSize) / int(packetSize)) * int(packetSize),
+      size = DstXprType::SizeAtCompileTime
+    };
+
+    for (Index outer = 0; outer < kernel.outerSize(); ++outer) {
+      copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, vectorizableSize, 0, 0>::run(kernel, outer);
+      copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, vectorizableSize, innerSize>::run(kernel, outer);
+    }
+  }
+};
+#endif
+
+/***************************************************************************
+ * Part 4 : Generic dense assignment kernel
+ ***************************************************************************/
+
+// This class generalize the assignment of a coefficient (or packet) from one dense evaluator
+// to another dense writable evaluator.
+// It is parametrized by the two evaluators, and the actual assignment functor.
+// This abstraction level permits to keep the evaluation loops as simple and as generic as possible.
+// One can customize the assignment using this generic dense_assignment_kernel with different
+// functors, or by completely overloading it, by-passing a functor.
+template <typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor, int Version = Specialized>
+class generic_dense_assignment_kernel {
+ protected:
+  typedef typename DstEvaluatorTypeT::XprType DstXprType;
+  typedef typename SrcEvaluatorTypeT::XprType SrcXprType;
+
+ public:
+  typedef DstEvaluatorTypeT DstEvaluatorType;
+  typedef SrcEvaluatorTypeT SrcEvaluatorType;
+  typedef typename DstEvaluatorType::Scalar Scalar;
+  typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor> AssignmentTraits;
+  typedef typename AssignmentTraits::PacketType PacketType;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE generic_dense_assignment_kernel(DstEvaluatorType& dst,
+                                                                        const SrcEvaluatorType& src,
+                                                                        const Functor& func, DstXprType& dstExpr)
+      : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr) {
+#ifdef EIGEN_DEBUG_ASSIGN
+    AssignmentTraits::debug();
+#endif
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index size() const EIGEN_NOEXCEPT { return m_dstExpr.size(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index innerSize() const EIGEN_NOEXCEPT { return m_dstExpr.innerSize(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index outerSize() const EIGEN_NOEXCEPT { return m_dstExpr.outerSize(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_dstExpr.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_dstExpr.cols(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index outerStride() const EIGEN_NOEXCEPT { return m_dstExpr.outerStride(); }
+
+  EIGEN_DEVICE_FUNC DstEvaluatorType& dstEvaluator() EIGEN_NOEXCEPT { return m_dst; }
+  EIGEN_DEVICE_FUNC const SrcEvaluatorType& srcEvaluator() const EIGEN_NOEXCEPT { return m_src; }
+
+  /// Assign src(row,col) to dst(row,col) through the assignment functor.
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Index row, Index col) {
+    m_functor.assignCoeff(m_dst.coeffRef(row, col), m_src.coeff(row, col));
+  }
+
+  /// \sa assignCoeff(Index,Index)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Index index) {
+    m_functor.assignCoeff(m_dst.coeffRef(index), m_src.coeff(index));
+  }
+
+  /// \sa assignCoeff(Index,Index)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeffByOuterInner(Index outer, Index inner) {
+    Index row = rowIndexByOuterInner(outer, inner);
+    Index col = colIndexByOuterInner(outer, inner);
+    assignCoeff(row, col);
+  }
+
+  template <int StoreMode, int LoadMode, typename Packet>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index row, Index col) {
+    m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(row, col),
+                                               m_src.template packet<LoadMode, Packet>(row, col));
+  }
+
+  template <int StoreMode, int LoadMode, typename Packet>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index index) {
+    m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(index), m_src.template packet<LoadMode, Packet>(index));
+  }
+
+  template <int StoreMode, int LoadMode, typename Packet>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner) {
+    Index row = rowIndexByOuterInner(outer, inner);
+    Index col = colIndexByOuterInner(outer, inner);
+    assignPacket<StoreMode, LoadMode, Packet>(row, col);
+  }
+
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index rowIndexByOuterInner(Index outer, Index inner) {
+    typedef typename DstEvaluatorType::ExpressionTraits Traits;
+    return int(Traits::RowsAtCompileTime) == 1          ? 0
+           : int(Traits::ColsAtCompileTime) == 1        ? inner
+           : int(DstEvaluatorType::Flags) & RowMajorBit ? outer
+                                                        : inner;
+  }
+
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index colIndexByOuterInner(Index outer, Index inner) {
+    typedef typename DstEvaluatorType::ExpressionTraits Traits;
+    return int(Traits::ColsAtCompileTime) == 1          ? 0
+           : int(Traits::RowsAtCompileTime) == 1        ? inner
+           : int(DstEvaluatorType::Flags) & RowMajorBit ? inner
+                                                        : outer;
+  }
+
+  EIGEN_DEVICE_FUNC const Scalar* dstDataPtr() const { return m_dstExpr.data(); }
+
+ protected:
+  DstEvaluatorType& m_dst;
+  const SrcEvaluatorType& m_src;
+  const Functor& m_functor;
+  // TODO find a way to avoid the needs of the original expression
+  DstXprType& m_dstExpr;
+};
+
+// Special kernel used when computing small products whose operands have dynamic dimensions.  It ensures that the
+// PacketSize used is no larger than 4, thereby increasing the chance that vectorized instructions will be used
+// when computing the product.
+
+template <typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor>
+class restricted_packet_dense_assignment_kernel
+    : public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn> {
+ protected:
+  typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn> Base;
+
+ public:
+  typedef typename Base::Scalar Scalar;
+  typedef typename Base::DstXprType DstXprType;
+  typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, 4> AssignmentTraits;
+  typedef typename AssignmentTraits::PacketType PacketType;
+
+  EIGEN_DEVICE_FUNC restricted_packet_dense_assignment_kernel(DstEvaluatorTypeT& dst, const SrcEvaluatorTypeT& src,
+                                                              const Functor& func, DstXprType& dstExpr)
+      : Base(dst, src, func, dstExpr) {}
+};
+
+/***************************************************************************
+ * Part 5 : Entry point for dense rectangular assignment
+ ***************************************************************************/
+
+template <typename DstXprType, typename SrcXprType, typename Functor>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resize_if_allowed(DstXprType& dst, const SrcXprType& src,
+                                                             const Functor& /*func*/) {
+  EIGEN_ONLY_USED_FOR_DEBUG(dst);
+  EIGEN_ONLY_USED_FOR_DEBUG(src);
+  eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
+}
+
+template <typename DstXprType, typename SrcXprType, typename T1, typename T2>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resize_if_allowed(DstXprType& dst, const SrcXprType& src,
+                                                             const internal::assign_op<T1, T2>& /*func*/) {
+  Index dstRows = src.rows();
+  Index dstCols = src.cols();
+  if (((dst.rows() != dstRows) || (dst.cols() != dstCols))) dst.resize(dstRows, dstCols);
+  eigen_assert(dst.rows() == dstRows && dst.cols() == dstCols);
+}
+
+template <typename DstXprType, typename SrcXprType, typename Functor>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_dense_assignment_loop(DstXprType& dst,
+                                                                                      const SrcXprType& src,
+                                                                                      const Functor& func) {
+  typedef evaluator<DstXprType> DstEvaluatorType;
+  typedef evaluator<SrcXprType> SrcEvaluatorType;
+
+  SrcEvaluatorType srcEvaluator(src);
+
+  // NOTE To properly handle A = (A*A.transpose())/s with A rectangular,
+  // we need to resize the destination after the source evaluator has been created.
+  resize_if_allowed(dst, src, func);
+
+  DstEvaluatorType dstEvaluator(dst);
+
+  typedef generic_dense_assignment_kernel<DstEvaluatorType, SrcEvaluatorType, Functor> Kernel;
+  Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
+
+  dense_assignment_loop<Kernel>::run(kernel);
+}
+
+template <typename DstXprType, typename SrcXprType>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(DstXprType& dst, const SrcXprType& src) {
+  call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>());
+}
+
+/***************************************************************************
+ * Part 6 : Generic assignment
+ ***************************************************************************/
+
+// Based on the respective shapes of the destination and source,
+// the class AssignmentKind determine the kind of assignment mechanism.
+// AssignmentKind must define a Kind typedef.
+template <typename DstShape, typename SrcShape>
+struct AssignmentKind;
+
+// Assignment kind defined in this file:
+struct Dense2Dense {};
+struct EigenBase2EigenBase {};
+
+template <typename, typename>
+struct AssignmentKind {
+  typedef EigenBase2EigenBase Kind;
+};
+template <>
+struct AssignmentKind<DenseShape, DenseShape> {
+  typedef Dense2Dense Kind;
+};
+
+// This is the main assignment class
+template <typename DstXprType, typename SrcXprType, typename Functor,
+          typename Kind = typename AssignmentKind<typename evaluator_traits<DstXprType>::Shape,
+                                                  typename evaluator_traits<SrcXprType>::Shape>::Kind,
+          typename EnableIf = void>
+struct Assignment;
+
+// The only purpose of this call_assignment() function is to deal with noalias() / "assume-aliasing" and automatic
+// transposition. Indeed, I (Gael) think that this concept of "assume-aliasing" was a mistake, and it makes thing quite
+// complicated. So this intermediate function removes everything related to "assume-aliasing" such that Assignment does
+// not has to bother about these annoying details.
+
+template <typename Dst, typename Src>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment(Dst& dst, const Src& src) {
+  call_assignment(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
+}
+template <typename Dst, typename Src>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment(const Dst& dst, const Src& src) {
+  call_assignment(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
+}
+
+// Deal with "assume-aliasing"
+template <typename Dst, typename Src, typename Func>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment(
+    Dst& dst, const Src& src, const Func& func, std::enable_if_t<evaluator_assume_aliasing<Src>::value, void*> = 0) {
+  typename plain_matrix_type<Src>::type tmp(src);
+  call_assignment_no_alias(dst, tmp, func);
+}
+
+template <typename Dst, typename Src, typename Func>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment(
+    Dst& dst, const Src& src, const Func& func, std::enable_if_t<!evaluator_assume_aliasing<Src>::value, void*> = 0) {
+  call_assignment_no_alias(dst, src, func);
+}
+
+// by-pass "assume-aliasing"
+// When there is no aliasing, we require that 'dst' has been properly resized
+template <typename Dst, template <typename> class StorageBase, typename Src, typename Func>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment(NoAlias<Dst, StorageBase>& dst,
+                                                                           const Src& src, const Func& func) {
+  call_assignment_no_alias(dst.expression(), src, func);
+}
+
+template <typename Dst, typename Src, typename Func>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment_no_alias(Dst& dst, const Src& src,
+                                                                                    const Func& func) {
+  enum {
+    NeedToTranspose = ((int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1) ||
+                       (int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)) &&
+                      int(Dst::SizeAtCompileTime) != 1
+  };
+
+  typedef std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst> ActualDstTypeCleaned;
+  typedef std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst&> ActualDstType;
+  ActualDstType actualDst(dst);
+
+  // TODO check whether this is the right place to perform these checks:
+  EIGEN_STATIC_ASSERT_LVALUE(Dst)
+  EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned, Src)
+  EIGEN_CHECK_BINARY_COMPATIBILIY(Func, typename ActualDstTypeCleaned::Scalar, typename Src::Scalar);
+
+  Assignment<ActualDstTypeCleaned, Src, Func>::run(actualDst, src, func);
+}
+
+template <typename Dst, typename Src, typename Func>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_restricted_packet_assignment_no_alias(Dst& dst, const Src& src,
+                                                                                      const Func& func) {
+  typedef evaluator<Dst> DstEvaluatorType;
+  typedef evaluator<Src> SrcEvaluatorType;
+  typedef restricted_packet_dense_assignment_kernel<DstEvaluatorType, SrcEvaluatorType, Func> Kernel;
+
+  EIGEN_STATIC_ASSERT_LVALUE(Dst)
+  EIGEN_CHECK_BINARY_COMPATIBILIY(Func, typename Dst::Scalar, typename Src::Scalar);
+
+  SrcEvaluatorType srcEvaluator(src);
+  resize_if_allowed(dst, src, func);
+
+  DstEvaluatorType dstEvaluator(dst);
+  Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
+
+  dense_assignment_loop<Kernel>::run(kernel);
+}
+
+template <typename Dst, typename Src>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment_no_alias(Dst& dst, const Src& src) {
+  call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
+}
+
+template <typename Dst, typename Src, typename Func>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment_no_alias_no_transpose(Dst& dst,
+                                                                                                 const Src& src,
+                                                                                                 const Func& func) {
+  // TODO check whether this is the right place to perform these checks:
+  EIGEN_STATIC_ASSERT_LVALUE(Dst)
+  EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst, Src)
+  EIGEN_CHECK_BINARY_COMPATIBILIY(Func, typename Dst::Scalar, typename Src::Scalar);
+
+  Assignment<Dst, Src, Func>::run(dst, src, func);
+}
+template <typename Dst, typename Src>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment_no_alias_no_transpose(Dst& dst,
+                                                                                                 const Src& src) {
+  call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
+}
+
+// forward declaration
+template <typename Dst, typename Src>
+EIGEN_DEVICE_FUNC void check_for_aliasing(const Dst& dst, const Src& src);
+
+// Generic Dense to Dense assignment
+// Note that the last template argument "Weak" is needed to make it possible to perform
+// both partial specialization+SFINAE without ambiguous specialization
+template <typename DstXprType, typename SrcXprType, typename Functor, typename Weak>
+struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Weak> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src, const Functor& func) {
+#ifndef EIGEN_NO_DEBUG
+    internal::check_for_aliasing(dst, src);
+#endif
+
+    call_dense_assignment_loop(dst, src, func);
+  }
+};
+
+template <typename DstXprType, typename SrcPlainObject, typename Weak>
+struct Assignment<DstXprType, CwiseNullaryOp<scalar_constant_op<typename DstXprType::Scalar>, SrcPlainObject>,
+                  assign_op<typename DstXprType::Scalar, typename DstXprType::Scalar>, Dense2Dense, Weak> {
+  using Scalar = typename DstXprType::Scalar;
+  using NullaryOp = scalar_constant_op<Scalar>;
+  using SrcXprType = CwiseNullaryOp<NullaryOp, SrcPlainObject>;
+  using Functor = assign_op<Scalar, Scalar>;
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src,
+                                                        const Functor& /*func*/) {
+    eigen_fill_impl<DstXprType>::run(dst, src);
+  }
+};
+
+template <typename DstXprType, typename SrcPlainObject, typename Weak>
+struct Assignment<DstXprType, CwiseNullaryOp<scalar_zero_op<typename DstXprType::Scalar>, SrcPlainObject>,
+                  assign_op<typename DstXprType::Scalar, typename DstXprType::Scalar>, Dense2Dense, Weak> {
+  using Scalar = typename DstXprType::Scalar;
+  using NullaryOp = scalar_zero_op<Scalar>;
+  using SrcXprType = CwiseNullaryOp<NullaryOp, SrcPlainObject>;
+  using Functor = assign_op<Scalar, Scalar>;
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src,
+                                                        const Functor& /*func*/) {
+    eigen_zero_impl<DstXprType>::run(dst, src);
+  }
+};
+
+// Generic assignment through evalTo.
+// TODO: not sure we have to keep that one, but it helps porting current code to new evaluator mechanism.
+// Note that the last template argument "Weak" is needed to make it possible to perform
+// both partial specialization+SFINAE without ambiguous specialization
+template <typename DstXprType, typename SrcXprType, typename Functor, typename Weak>
+struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Weak> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(
+      DstXprType& dst, const SrcXprType& src,
+      const internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>& /*func*/) {
+    Index dstRows = src.rows();
+    Index dstCols = src.cols();
+    if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
+
+    eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
+    src.evalTo(dst);
+  }
+
+  // NOTE The following two functions are templated to avoid their instantiation if not needed
+  //      This is needed because some expressions supports evalTo only and/or have 'void' as scalar type.
+  template <typename SrcScalarType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(
+      DstXprType& dst, const SrcXprType& src,
+      const internal::add_assign_op<typename DstXprType::Scalar, SrcScalarType>& /*func*/) {
+    Index dstRows = src.rows();
+    Index dstCols = src.cols();
+    if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
+
+    eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
+    src.addTo(dst);
+  }
+
+  template <typename SrcScalarType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(
+      DstXprType& dst, const SrcXprType& src,
+      const internal::sub_assign_op<typename DstXprType::Scalar, SrcScalarType>& /*func*/) {
+    Index dstRows = src.rows();
+    Index dstCols = src.cols();
+    if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
+
+    eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
+    src.subTo(dst);
+  }
+};
+
+}  // namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_ASSIGN_EVALUATOR_H

include/Eigen/src/Core/Assign_MKL.h

@@ -0,0 +1,183 @@
+/*
+ Copyright (c) 2011, Intel Corporation. All rights reserved.
+ Copyright (C) 2015 Gael Guennebaud <gael.guennebaud@inria.fr>
+
+ Redistribution and use in source and binary forms, with or without modification,
+ are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+ * Neither the name of Intel Corporation nor the names of its contributors may
+   be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ ********************************************************************************
+ *   Content : Eigen bindings to Intel(R) MKL
+ *   MKL VML support for coefficient-wise unary Eigen expressions like a=b.sin()
+ ********************************************************************************
+*/
+
+#ifndef EIGEN_ASSIGN_VML_H
+#define EIGEN_ASSIGN_VML_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Dst, typename Src>
+class vml_assign_traits {
+ private:
+  enum {
+    DstHasDirectAccess = Dst::Flags & DirectAccessBit,
+    SrcHasDirectAccess = Src::Flags & DirectAccessBit,
+    StorageOrdersAgree = (int(Dst::IsRowMajor) == int(Src::IsRowMajor)),
+    InnerSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::SizeAtCompileTime)
+                : int(Dst::Flags) & RowMajorBit ? int(Dst::ColsAtCompileTime)
+                                                : int(Dst::RowsAtCompileTime),
+    InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime)
+                   : int(Dst::Flags) & RowMajorBit ? int(Dst::MaxColsAtCompileTime)
+                                                   : int(Dst::MaxRowsAtCompileTime),
+    MaxSizeAtCompileTime = Dst::SizeAtCompileTime,
+
+    MightEnableVml = StorageOrdersAgree && DstHasDirectAccess && SrcHasDirectAccess &&
+                     Src::InnerStrideAtCompileTime == 1 && Dst::InnerStrideAtCompileTime == 1,
+    MightLinearize = MightEnableVml && (int(Dst::Flags) & int(Src::Flags) & LinearAccessBit),
+    VmlSize = MightLinearize ? MaxSizeAtCompileTime : InnerMaxSize,
+    LargeEnough = VmlSize == Dynamic || VmlSize >= EIGEN_MKL_VML_THRESHOLD
+  };
+
+ public:
+  enum { EnableVml = MightEnableVml && LargeEnough, Traversal = MightLinearize ? LinearTraversal : DefaultTraversal };
+};
+
+#define EIGEN_PP_EXPAND(ARG) ARG
+#if !defined(EIGEN_FAST_MATH) || (EIGEN_FAST_MATH != 1)
+#define EIGEN_VMLMODE_EXPAND_xLA , VML_HA
+#else
+#define EIGEN_VMLMODE_EXPAND_xLA , VML_LA
+#endif
+
+#define EIGEN_VMLMODE_EXPAND_x_
+
+#define EIGEN_VMLMODE_PREFIX_xLA vm
+#define EIGEN_VMLMODE_PREFIX_x_ v
+#define EIGEN_VMLMODE_PREFIX(VMLMODE) EIGEN_CAT(EIGEN_VMLMODE_PREFIX_x, VMLMODE)
+
+#define EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, VMLOP, EIGENTYPE, VMLTYPE, VMLMODE)                      \
+  template <typename DstXprType, typename SrcXprNested>                                                    \
+  struct Assignment<DstXprType, CwiseUnaryOp<scalar_##EIGENOP##_op<EIGENTYPE>, SrcXprNested>,              \
+                    assign_op<EIGENTYPE, EIGENTYPE>, Dense2Dense,                                          \
+                    std::enable_if_t<vml_assign_traits<DstXprType, SrcXprNested>::EnableVml>> {            \
+    typedef CwiseUnaryOp<scalar_##EIGENOP##_op<EIGENTYPE>, SrcXprNested> SrcXprType;                       \
+    static void run(DstXprType &dst, const SrcXprType &src, const assign_op<EIGENTYPE, EIGENTYPE> &func) { \
+      resize_if_allowed(dst, src, func);                                                                   \
+      eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());                                  \
+      if (vml_assign_traits<DstXprType, SrcXprNested>::Traversal == (int)LinearTraversal) {                \
+        VMLOP(dst.size(), (const VMLTYPE *)src.nestedExpression().data(),                                  \
+              (VMLTYPE *)dst.data() EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_x##VMLMODE));                     \
+      } else {                                                                                             \
+        const Index outerSize = dst.outerSize();                                                           \
+        for (Index outer = 0; outer < outerSize; ++outer) {                                                \
+          const EIGENTYPE *src_ptr = src.IsRowMajor ? &(src.nestedExpression().coeffRef(outer, 0))         \
+                                                    : &(src.nestedExpression().coeffRef(0, outer));        \
+          EIGENTYPE *dst_ptr = dst.IsRowMajor ? &(dst.coeffRef(outer, 0)) : &(dst.coeffRef(0, outer));     \
+          VMLOP(dst.innerSize(), (const VMLTYPE *)src_ptr,                                                 \
+                (VMLTYPE *)dst_ptr EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_x##VMLMODE));                      \
+        }                                                                                                  \
+      }                                                                                                    \
+    }                                                                                                      \
+  };
+
+#define EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(EIGENOP, VMLOP, VMLMODE)                                                \
+  EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, EIGEN_CAT(EIGEN_VMLMODE_PREFIX(VMLMODE), s##VMLOP), float, float, VMLMODE) \
+  EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, EIGEN_CAT(EIGEN_VMLMODE_PREFIX(VMLMODE), d##VMLOP), double, double, VMLMODE)
+
+#define EIGEN_MKL_VML_DECLARE_UNARY_CALLS_CPLX(EIGENOP, VMLOP, VMLMODE)                                   \
+  EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, EIGEN_CAT(EIGEN_VMLMODE_PREFIX(VMLMODE), c##VMLOP), scomplex, \
+                                   MKL_Complex8, VMLMODE)                                                 \
+  EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, EIGEN_CAT(EIGEN_VMLMODE_PREFIX(VMLMODE), z##VMLOP), dcomplex, \
+                                   MKL_Complex16, VMLMODE)
+
+#define EIGEN_MKL_VML_DECLARE_UNARY_CALLS(EIGENOP, VMLOP, VMLMODE) \
+  EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(EIGENOP, VMLOP, VMLMODE)  \
+  EIGEN_MKL_VML_DECLARE_UNARY_CALLS_CPLX(EIGENOP, VMLOP, VMLMODE)
+
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(sin, Sin, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(asin, Asin, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(sinh, Sinh, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(cos, Cos, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(acos, Acos, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(cosh, Cosh, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(tan, Tan, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(atan, Atan, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(tanh, Tanh, LA)
+// EIGEN_MKL_VML_DECLARE_UNARY_CALLS(abs,   Abs,    _)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(exp, Exp, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(log, Ln, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(log10, Log10, LA)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS(sqrt, Sqrt, _)
+
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(square, Sqr, _)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS_CPLX(arg, Arg, _)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(round, Round, _)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(floor, Floor, _)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(ceil, Ceil, _)
+EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(cbrt, Cbrt, _)
+
+#define EIGEN_MKL_VML_DECLARE_POW_CALL(EIGENOP, VMLOP, EIGENTYPE, VMLTYPE, VMLMODE)                        \
+  template <typename DstXprType, typename SrcXprNested, typename Plain>                                    \
+  struct Assignment<DstXprType,                                                                            \
+                    CwiseBinaryOp<scalar_##EIGENOP##_op<EIGENTYPE, EIGENTYPE>, SrcXprNested,               \
+                                  const CwiseNullaryOp<internal::scalar_constant_op<EIGENTYPE>, Plain>>,   \
+                    assign_op<EIGENTYPE, EIGENTYPE>, Dense2Dense,                                          \
+                    std::enable_if_t<vml_assign_traits<DstXprType, SrcXprNested>::EnableVml>> {            \
+    typedef CwiseBinaryOp<scalar_##EIGENOP##_op<EIGENTYPE, EIGENTYPE>, SrcXprNested,                       \
+                          const CwiseNullaryOp<internal::scalar_constant_op<EIGENTYPE>, Plain>>            \
+        SrcXprType;                                                                                        \
+    static void run(DstXprType &dst, const SrcXprType &src, const assign_op<EIGENTYPE, EIGENTYPE> &func) { \
+      resize_if_allowed(dst, src, func);                                                                   \
+      eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());                                  \
+      VMLTYPE exponent = reinterpret_cast<const VMLTYPE &>(src.rhs().functor().m_other);                   \
+      if (vml_assign_traits<DstXprType, SrcXprNested>::Traversal == LinearTraversal) {                     \
+        VMLOP(dst.size(), (const VMLTYPE *)src.lhs().data(), exponent,                                     \
+              (VMLTYPE *)dst.data() EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_x##VMLMODE));                     \
+      } else {                                                                                             \
+        const Index outerSize = dst.outerSize();                                                           \
+        for (Index outer = 0; outer < outerSize; ++outer) {                                                \
+          const EIGENTYPE *src_ptr =                                                                       \
+              src.IsRowMajor ? &(src.lhs().coeffRef(outer, 0)) : &(src.lhs().coeffRef(0, outer));          \
+          EIGENTYPE *dst_ptr = dst.IsRowMajor ? &(dst.coeffRef(outer, 0)) : &(dst.coeffRef(0, outer));     \
+          VMLOP(dst.innerSize(), (const VMLTYPE *)src_ptr, exponent,                                       \
+                (VMLTYPE *)dst_ptr EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_x##VMLMODE));                      \
+        }                                                                                                  \
+      }                                                                                                    \
+    }                                                                                                      \
+  };
+
+EIGEN_MKL_VML_DECLARE_POW_CALL(pow, vmsPowx, float, float, LA)
+EIGEN_MKL_VML_DECLARE_POW_CALL(pow, vmdPowx, double, double, LA)
+EIGEN_MKL_VML_DECLARE_POW_CALL(pow, vmcPowx, scomplex, MKL_Complex8, LA)
+EIGEN_MKL_VML_DECLARE_POW_CALL(pow, vmzPowx, dcomplex, MKL_Complex16, LA)
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_ASSIGN_VML_H

include/Eigen/src/Core/BandMatrix.h

@@ -0,0 +1,338 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_BANDMATRIX_H
+#define EIGEN_BANDMATRIX_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Derived>
+class BandMatrixBase : public EigenBase<Derived> {
+ public:
+  enum {
+    Flags = internal::traits<Derived>::Flags,
+    CoeffReadCost = internal::traits<Derived>::CoeffReadCost,
+    RowsAtCompileTime = internal::traits<Derived>::RowsAtCompileTime,
+    ColsAtCompileTime = internal::traits<Derived>::ColsAtCompileTime,
+    MaxRowsAtCompileTime = internal::traits<Derived>::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = internal::traits<Derived>::MaxColsAtCompileTime,
+    Supers = internal::traits<Derived>::Supers,
+    Subs = internal::traits<Derived>::Subs,
+    Options = internal::traits<Derived>::Options
+  };
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+  typedef Matrix<Scalar, RowsAtCompileTime, ColsAtCompileTime> DenseMatrixType;
+  typedef typename DenseMatrixType::StorageIndex StorageIndex;
+  typedef typename internal::traits<Derived>::CoefficientsType CoefficientsType;
+  typedef EigenBase<Derived> Base;
+
+ protected:
+  enum {
+    DataRowsAtCompileTime = ((Supers != Dynamic) && (Subs != Dynamic)) ? 1 + Supers + Subs : Dynamic,
+    SizeAtCompileTime = min_size_prefer_dynamic(RowsAtCompileTime, ColsAtCompileTime)
+  };
+
+ public:
+  using Base::cols;
+  using Base::derived;
+  using Base::rows;
+
+  /** \returns the number of super diagonals */
+  inline Index supers() const { return derived().supers(); }
+
+  /** \returns the number of sub diagonals */
+  inline Index subs() const { return derived().subs(); }
+
+  /** \returns an expression of the underlying coefficient matrix */
+  inline const CoefficientsType& coeffs() const { return derived().coeffs(); }
+
+  /** \returns an expression of the underlying coefficient matrix */
+  inline CoefficientsType& coeffs() { return derived().coeffs(); }
+
+  /** \returns a vector expression of the \a i -th column,
+   * only the meaningful part is returned.
+   * \warning the internal storage must be column major. */
+  inline Block<CoefficientsType, Dynamic, 1> col(Index i) {
+    EIGEN_STATIC_ASSERT((int(Options) & int(RowMajor)) == 0, THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES);
+    Index start = 0;
+    Index len = coeffs().rows();
+    if (i <= supers()) {
+      start = supers() - i;
+      len = (std::min)(rows(), std::max<Index>(0, coeffs().rows() - (supers() - i)));
+    } else if (i >= rows() - subs())
+      len = std::max<Index>(0, coeffs().rows() - (i + 1 - rows() + subs()));
+    return Block<CoefficientsType, Dynamic, 1>(coeffs(), start, i, len, 1);
+  }
+
+  /** \returns a vector expression of the main diagonal */
+  inline Block<CoefficientsType, 1, SizeAtCompileTime> diagonal() {
+    return Block<CoefficientsType, 1, SizeAtCompileTime>(coeffs(), supers(), 0, 1, (std::min)(rows(), cols()));
+  }
+
+  /** \returns a vector expression of the main diagonal (const version) */
+  inline const Block<const CoefficientsType, 1, SizeAtCompileTime> diagonal() const {
+    return Block<const CoefficientsType, 1, SizeAtCompileTime>(coeffs(), supers(), 0, 1, (std::min)(rows(), cols()));
+  }
+
+  template <int Index>
+  struct DiagonalIntReturnType {
+    enum {
+      ReturnOpposite =
+          (int(Options) & int(SelfAdjoint)) && (((Index) > 0 && Supers == 0) || ((Index) < 0 && Subs == 0)),
+      Conjugate = ReturnOpposite && NumTraits<Scalar>::IsComplex,
+      ActualIndex = ReturnOpposite ? -Index : Index,
+      DiagonalSize =
+          (RowsAtCompileTime == Dynamic || ColsAtCompileTime == Dynamic)
+              ? Dynamic
+              : (ActualIndex < 0 ? min_size_prefer_dynamic(ColsAtCompileTime, RowsAtCompileTime + ActualIndex)
+                                 : min_size_prefer_dynamic(RowsAtCompileTime, ColsAtCompileTime - ActualIndex))
+    };
+    typedef Block<CoefficientsType, 1, DiagonalSize> BuildType;
+    typedef std::conditional_t<Conjugate, CwiseUnaryOp<internal::scalar_conjugate_op<Scalar>, BuildType>, BuildType>
+        Type;
+  };
+
+  /** \returns a vector expression of the \a N -th sub or super diagonal */
+  template <int N>
+  inline typename DiagonalIntReturnType<N>::Type diagonal() {
+    return typename DiagonalIntReturnType<N>::BuildType(coeffs(), supers() - N, (std::max)(0, N), 1, diagonalLength(N));
+  }
+
+  /** \returns a vector expression of the \a N -th sub or super diagonal */
+  template <int N>
+  inline const typename DiagonalIntReturnType<N>::Type diagonal() const {
+    return typename DiagonalIntReturnType<N>::BuildType(coeffs(), supers() - N, (std::max)(0, N), 1, diagonalLength(N));
+  }
+
+  /** \returns a vector expression of the \a i -th sub or super diagonal */
+  inline Block<CoefficientsType, 1, Dynamic> diagonal(Index i) {
+    eigen_assert((i < 0 && -i <= subs()) || (i >= 0 && i <= supers()));
+    return Block<CoefficientsType, 1, Dynamic>(coeffs(), supers() - i, std::max<Index>(0, i), 1, diagonalLength(i));
+  }
+
+  /** \returns a vector expression of the \a i -th sub or super diagonal */
+  inline const Block<const CoefficientsType, 1, Dynamic> diagonal(Index i) const {
+    eigen_assert((i < 0 && -i <= subs()) || (i >= 0 && i <= supers()));
+    return Block<const CoefficientsType, 1, Dynamic>(coeffs(), supers() - i, std::max<Index>(0, i), 1,
+                                                     diagonalLength(i));
+  }
+
+  template <typename Dest>
+  inline void evalTo(Dest& dst) const {
+    dst.resize(rows(), cols());
+    dst.setZero();
+    dst.diagonal() = diagonal();
+    for (Index i = 1; i <= supers(); ++i) dst.diagonal(i) = diagonal(i);
+    for (Index i = 1; i <= subs(); ++i) dst.diagonal(-i) = diagonal(-i);
+  }
+
+  DenseMatrixType toDenseMatrix() const {
+    DenseMatrixType res(rows(), cols());
+    evalTo(res);
+    return res;
+  }
+
+ protected:
+  inline Index diagonalLength(Index i) const {
+    return i < 0 ? (std::min)(cols(), rows() + i) : (std::min)(rows(), cols() - i);
+  }
+};
+
+/**
+ * \class BandMatrix
+ * \ingroup Core_Module
+ *
+ * \brief Represents a rectangular matrix with a banded storage
+ *
+ * \tparam Scalar_ Numeric type, i.e. float, double, int
+ * \tparam Rows_ Number of rows, or \b Dynamic
+ * \tparam Cols_ Number of columns, or \b Dynamic
+ * \tparam Supers_ Number of super diagonal
+ * \tparam Subs_ Number of sub diagonal
+ * \tparam Options_ A combination of either \b #RowMajor or \b #ColMajor, and of \b #SelfAdjoint
+ *                  The former controls \ref TopicStorageOrders "storage order", and defaults to
+ *                  column-major. The latter controls whether the matrix represents a selfadjoint
+ *                  matrix in which case either Supers of Subs have to be null.
+ *
+ * \sa class TridiagonalMatrix
+ */
+
+template <typename Scalar_, int Rows_, int Cols_, int Supers_, int Subs_, int Options_>
+struct traits<BandMatrix<Scalar_, Rows_, Cols_, Supers_, Subs_, Options_> > {
+  typedef Scalar_ Scalar;
+  typedef Dense StorageKind;
+  typedef Eigen::Index StorageIndex;
+  enum {
+    CoeffReadCost = NumTraits<Scalar>::ReadCost,
+    RowsAtCompileTime = Rows_,
+    ColsAtCompileTime = Cols_,
+    MaxRowsAtCompileTime = Rows_,
+    MaxColsAtCompileTime = Cols_,
+    Flags = LvalueBit,
+    Supers = Supers_,
+    Subs = Subs_,
+    Options = Options_,
+    DataRowsAtCompileTime = ((Supers != Dynamic) && (Subs != Dynamic)) ? 1 + Supers + Subs : Dynamic
+  };
+  typedef Matrix<Scalar, DataRowsAtCompileTime, ColsAtCompileTime, int(Options) & int(RowMajor) ? RowMajor : ColMajor>
+      CoefficientsType;
+};
+
+template <typename Scalar_, int Rows, int Cols, int Supers, int Subs, int Options>
+class BandMatrix : public BandMatrixBase<BandMatrix<Scalar_, Rows, Cols, Supers, Subs, Options> > {
+ public:
+  typedef typename internal::traits<BandMatrix>::Scalar Scalar;
+  typedef typename internal::traits<BandMatrix>::StorageIndex StorageIndex;
+  typedef typename internal::traits<BandMatrix>::CoefficientsType CoefficientsType;
+
+  explicit inline BandMatrix(Index rows = Rows, Index cols = Cols, Index supers = Supers, Index subs = Subs)
+      : m_coeffs(1 + supers + subs, cols), m_rows(rows), m_supers(supers), m_subs(subs) {}
+
+  /** \returns the number of columns */
+  inline EIGEN_CONSTEXPR Index rows() const { return m_rows.value(); }
+
+  /** \returns the number of rows */
+  inline EIGEN_CONSTEXPR Index cols() const { return m_coeffs.cols(); }
+
+  /** \returns the number of super diagonals */
+  inline EIGEN_CONSTEXPR Index supers() const { return m_supers.value(); }
+
+  /** \returns the number of sub diagonals */
+  inline EIGEN_CONSTEXPR Index subs() const { return m_subs.value(); }
+
+  inline const CoefficientsType& coeffs() const { return m_coeffs; }
+  inline CoefficientsType& coeffs() { return m_coeffs; }
+
+ protected:
+  CoefficientsType m_coeffs;
+  internal::variable_if_dynamic<Index, Rows> m_rows;
+  internal::variable_if_dynamic<Index, Supers> m_supers;
+  internal::variable_if_dynamic<Index, Subs> m_subs;
+};
+
+template <typename CoefficientsType_, int Rows_, int Cols_, int Supers_, int Subs_, int Options_>
+class BandMatrixWrapper;
+
+template <typename CoefficientsType_, int Rows_, int Cols_, int Supers_, int Subs_, int Options_>
+struct traits<BandMatrixWrapper<CoefficientsType_, Rows_, Cols_, Supers_, Subs_, Options_> > {
+  typedef typename CoefficientsType_::Scalar Scalar;
+  typedef typename CoefficientsType_::StorageKind StorageKind;
+  typedef typename CoefficientsType_::StorageIndex StorageIndex;
+  enum {
+    CoeffReadCost = internal::traits<CoefficientsType_>::CoeffReadCost,
+    RowsAtCompileTime = Rows_,
+    ColsAtCompileTime = Cols_,
+    MaxRowsAtCompileTime = Rows_,
+    MaxColsAtCompileTime = Cols_,
+    Flags = LvalueBit,
+    Supers = Supers_,
+    Subs = Subs_,
+    Options = Options_,
+    DataRowsAtCompileTime = ((Supers != Dynamic) && (Subs != Dynamic)) ? 1 + Supers + Subs : Dynamic
+  };
+  typedef CoefficientsType_ CoefficientsType;
+};
+
+template <typename CoefficientsType_, int Rows_, int Cols_, int Supers_, int Subs_, int Options_>
+class BandMatrixWrapper
+    : public BandMatrixBase<BandMatrixWrapper<CoefficientsType_, Rows_, Cols_, Supers_, Subs_, Options_> > {
+ public:
+  typedef typename internal::traits<BandMatrixWrapper>::Scalar Scalar;
+  typedef typename internal::traits<BandMatrixWrapper>::CoefficientsType CoefficientsType;
+  typedef typename internal::traits<BandMatrixWrapper>::StorageIndex StorageIndex;
+
+  explicit inline BandMatrixWrapper(const CoefficientsType& coeffs, Index rows = Rows_, Index cols = Cols_,
+                                    Index supers = Supers_, Index subs = Subs_)
+      : m_coeffs(coeffs), m_rows(rows), m_supers(supers), m_subs(subs) {
+    EIGEN_UNUSED_VARIABLE(cols);
+    // eigen_assert(coeffs.cols()==cols() && (supers()+subs()+1)==coeffs.rows());
+  }
+
+  /** \returns the number of columns */
+  inline EIGEN_CONSTEXPR Index rows() const { return m_rows.value(); }
+
+  /** \returns the number of rows */
+  inline EIGEN_CONSTEXPR Index cols() const { return m_coeffs.cols(); }
+
+  /** \returns the number of super diagonals */
+  inline EIGEN_CONSTEXPR Index supers() const { return m_supers.value(); }
+
+  /** \returns the number of sub diagonals */
+  inline EIGEN_CONSTEXPR Index subs() const { return m_subs.value(); }
+
+  inline const CoefficientsType& coeffs() const { return m_coeffs; }
+
+ protected:
+  const CoefficientsType& m_coeffs;
+  internal::variable_if_dynamic<Index, Rows_> m_rows;
+  internal::variable_if_dynamic<Index, Supers_> m_supers;
+  internal::variable_if_dynamic<Index, Subs_> m_subs;
+};
+
+/**
+ * \class TridiagonalMatrix
+ * \ingroup Core_Module
+ *
+ * \brief Represents a tridiagonal matrix with a compact banded storage
+ *
+ * \tparam Scalar Numeric type, i.e. float, double, int
+ * \tparam Size Number of rows and cols, or \b Dynamic
+ * \tparam Options Can be 0 or \b SelfAdjoint
+ *
+ * \sa class BandMatrix
+ */
+template <typename Scalar, int Size, int Options>
+class TridiagonalMatrix : public BandMatrix<Scalar, Size, Size, Options & SelfAdjoint ? 0 : 1, 1, Options | RowMajor> {
+  typedef BandMatrix<Scalar, Size, Size, Options & SelfAdjoint ? 0 : 1, 1, Options | RowMajor> Base;
+  typedef typename Base::StorageIndex StorageIndex;
+
+ public:
+  explicit TridiagonalMatrix(Index size = Size) : Base(size, size, Options & SelfAdjoint ? 0 : 1, 1) {}
+
+  inline typename Base::template DiagonalIntReturnType<1>::Type super() { return Base::template diagonal<1>(); }
+  inline const typename Base::template DiagonalIntReturnType<1>::Type super() const {
+    return Base::template diagonal<1>();
+  }
+  inline typename Base::template DiagonalIntReturnType<-1>::Type sub() { return Base::template diagonal<-1>(); }
+  inline const typename Base::template DiagonalIntReturnType<-1>::Type sub() const {
+    return Base::template diagonal<-1>();
+  }
+
+ protected:
+};
+
+struct BandShape {};
+
+template <typename Scalar_, int Rows_, int Cols_, int Supers_, int Subs_, int Options_>
+struct evaluator_traits<BandMatrix<Scalar_, Rows_, Cols_, Supers_, Subs_, Options_> >
+    : public evaluator_traits_base<BandMatrix<Scalar_, Rows_, Cols_, Supers_, Subs_, Options_> > {
+  typedef BandShape Shape;
+};
+
+template <typename CoefficientsType_, int Rows_, int Cols_, int Supers_, int Subs_, int Options_>
+struct evaluator_traits<BandMatrixWrapper<CoefficientsType_, Rows_, Cols_, Supers_, Subs_, Options_> >
+    : public evaluator_traits_base<BandMatrixWrapper<CoefficientsType_, Rows_, Cols_, Supers_, Subs_, Options_> > {
+  typedef BandShape Shape;
+};
+
+template <>
+struct AssignmentKind<DenseShape, BandShape> {
+  typedef EigenBase2EigenBase Kind;
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_BANDMATRIX_H

include/Eigen/src/Core/Block.h

@@ -0,0 +1,439 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2006-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_BLOCK_H
+#define EIGEN_BLOCK_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename XprType_, int BlockRows, int BlockCols, bool InnerPanel_>
+struct traits<Block<XprType_, BlockRows, BlockCols, InnerPanel_>> : traits<XprType_> {
+  typedef typename traits<XprType_>::Scalar Scalar;
+  typedef typename traits<XprType_>::StorageKind StorageKind;
+  typedef typename traits<XprType_>::XprKind XprKind;
+  typedef typename ref_selector<XprType_>::type XprTypeNested;
+  typedef std::remove_reference_t<XprTypeNested> XprTypeNested_;
+  enum {
+    MatrixRows = traits<XprType_>::RowsAtCompileTime,
+    MatrixCols = traits<XprType_>::ColsAtCompileTime,
+    RowsAtCompileTime = MatrixRows == 0 ? 0 : BlockRows,
+    ColsAtCompileTime = MatrixCols == 0 ? 0 : BlockCols,
+    MaxRowsAtCompileTime = BlockRows == 0                 ? 0
+                           : RowsAtCompileTime != Dynamic ? int(RowsAtCompileTime)
+                                                          : int(traits<XprType_>::MaxRowsAtCompileTime),
+    MaxColsAtCompileTime = BlockCols == 0                 ? 0
+                           : ColsAtCompileTime != Dynamic ? int(ColsAtCompileTime)
+                                                          : int(traits<XprType_>::MaxColsAtCompileTime),
+
+    XprTypeIsRowMajor = (int(traits<XprType_>::Flags) & RowMajorBit) != 0,
+    IsRowMajor = (MaxRowsAtCompileTime == 1 && MaxColsAtCompileTime != 1)   ? 1
+                 : (MaxColsAtCompileTime == 1 && MaxRowsAtCompileTime != 1) ? 0
+                                                                            : XprTypeIsRowMajor,
+    HasSameStorageOrderAsXprType = (IsRowMajor == XprTypeIsRowMajor),
+    InnerSize = IsRowMajor ? int(ColsAtCompileTime) : int(RowsAtCompileTime),
+    InnerStrideAtCompileTime = HasSameStorageOrderAsXprType ? int(inner_stride_at_compile_time<XprType_>::ret)
+                                                            : int(outer_stride_at_compile_time<XprType_>::ret),
+    OuterStrideAtCompileTime = HasSameStorageOrderAsXprType ? int(outer_stride_at_compile_time<XprType_>::ret)
+                                                            : int(inner_stride_at_compile_time<XprType_>::ret),
+
+    // FIXME, this traits is rather specialized for dense object and it needs to be cleaned further
+    FlagsLvalueBit = is_lvalue<XprType_>::value ? LvalueBit : 0,
+    FlagsRowMajorBit = IsRowMajor ? RowMajorBit : 0,
+    Flags = (traits<XprType_>::Flags & (DirectAccessBit | (InnerPanel_ ? CompressedAccessBit : 0))) | FlagsLvalueBit |
+            FlagsRowMajorBit,
+    // FIXME DirectAccessBit should not be handled by expressions
+    //
+    // Alignment is needed by MapBase's assertions
+    // We can sefely set it to false here. Internal alignment errors will be detected by an eigen_internal_assert in the
+    // respective evaluator
+    Alignment = 0,
+    InnerPanel = InnerPanel_ ? 1 : 0
+  };
+};
+
+template <typename XprType, int BlockRows = Dynamic, int BlockCols = Dynamic, bool InnerPanel = false,
+          bool HasDirectAccess = internal::has_direct_access<XprType>::ret>
+class BlockImpl_dense;
+
+}  // end namespace internal
+
+template <typename XprType, int BlockRows, int BlockCols, bool InnerPanel, typename StorageKind>
+class BlockImpl;
+
+/** \class Block
+ * \ingroup Core_Module
+ *
+ * \brief Expression of a fixed-size or dynamic-size block
+ *
+ * \tparam XprType the type of the expression in which we are taking a block
+ * \tparam BlockRows the number of rows of the block we are taking at compile time (optional)
+ * \tparam BlockCols the number of columns of the block we are taking at compile time (optional)
+ * \tparam InnerPanel is true, if the block maps to a set of rows of a row major matrix or
+ *         to set of columns of a column major matrix (optional). The parameter allows to determine
+ *         at compile time whether aligned access is possible on the block expression.
+ *
+ * This class represents an expression of either a fixed-size or dynamic-size block. It is the return
+ * type of DenseBase::block(Index,Index,Index,Index) and DenseBase::block<int,int>(Index,Index) and
+ * most of the time this is the only way it is used.
+ *
+ * However, if you want to directly manipulate block expressions,
+ * for instance if you want to write a function returning such an expression, you
+ * will need to use this class.
+ *
+ * Here is an example illustrating the dynamic case:
+ * \include class_Block.cpp
+ * Output: \verbinclude class_Block.out
+ *
+ * \note Even though this expression has dynamic size, in the case where \a XprType
+ * has fixed size, this expression inherits a fixed maximal size which means that evaluating
+ * it does not cause a dynamic memory allocation.
+ *
+ * Here is an example illustrating the fixed-size case:
+ * \include class_FixedBlock.cpp
+ * Output: \verbinclude class_FixedBlock.out
+ *
+ * \sa DenseBase::block(Index,Index,Index,Index), DenseBase::block(Index,Index), class VectorBlock
+ */
+template <typename XprType, int BlockRows, int BlockCols, bool InnerPanel>
+class Block
+    : public BlockImpl<XprType, BlockRows, BlockCols, InnerPanel, typename internal::traits<XprType>::StorageKind> {
+  typedef BlockImpl<XprType, BlockRows, BlockCols, InnerPanel, typename internal::traits<XprType>::StorageKind> Impl;
+  using BlockHelper = internal::block_xpr_helper<Block>;
+
+ public:
+  // typedef typename Impl::Base Base;
+  typedef Impl Base;
+  EIGEN_GENERIC_PUBLIC_INTERFACE(Block)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Block)
+
+  typedef internal::remove_all_t<XprType> NestedExpression;
+
+  /** Column or Row constructor
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Block(XprType& xpr, Index i) : Impl(xpr, i) {
+    eigen_assert((i >= 0) && (((BlockRows == 1) && (BlockCols == XprType::ColsAtCompileTime) && i < xpr.rows()) ||
+                              ((BlockRows == XprType::RowsAtCompileTime) && (BlockCols == 1) && i < xpr.cols())));
+  }
+
+  /** Fixed-size constructor
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Block(XprType& xpr, Index startRow, Index startCol)
+      : Impl(xpr, startRow, startCol) {
+    EIGEN_STATIC_ASSERT(RowsAtCompileTime != Dynamic && ColsAtCompileTime != Dynamic,
+                        THIS_METHOD_IS_ONLY_FOR_FIXED_SIZE)
+    eigen_assert(startRow >= 0 && BlockRows >= 0 && startRow + BlockRows <= xpr.rows() && startCol >= 0 &&
+                 BlockCols >= 0 && startCol + BlockCols <= xpr.cols());
+  }
+
+  /** Dynamic-size constructor
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Block(XprType& xpr, Index startRow, Index startCol, Index blockRows,
+                                              Index blockCols)
+      : Impl(xpr, startRow, startCol, blockRows, blockCols) {
+    eigen_assert((RowsAtCompileTime == Dynamic || RowsAtCompileTime == blockRows) &&
+                 (ColsAtCompileTime == Dynamic || ColsAtCompileTime == blockCols));
+    eigen_assert(startRow >= 0 && blockRows >= 0 && startRow <= xpr.rows() - blockRows && startCol >= 0 &&
+                 blockCols >= 0 && startCol <= xpr.cols() - blockCols);
+  }
+
+  // convert nested blocks (e.g. Block<Block<MatrixType>>) to a simple block expression (Block<MatrixType>)
+
+  using ConstUnwindReturnType = Block<const typename BlockHelper::BaseType, BlockRows, BlockCols, InnerPanel>;
+  using UnwindReturnType = Block<typename BlockHelper::BaseType, BlockRows, BlockCols, InnerPanel>;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ConstUnwindReturnType unwind() const {
+    return ConstUnwindReturnType(BlockHelper::base(*this), BlockHelper::row(*this, 0), BlockHelper::col(*this, 0),
+                                 this->rows(), this->cols());
+  }
+
+  template <typename T = Block, typename EnableIf = std::enable_if_t<!std::is_const<T>::value>>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE UnwindReturnType unwind() {
+    return UnwindReturnType(BlockHelper::base(*this), BlockHelper::row(*this, 0), BlockHelper::col(*this, 0),
+                            this->rows(), this->cols());
+  }
+};
+
+// The generic default implementation for dense block simply forward to the internal::BlockImpl_dense
+// that must be specialized for direct and non-direct access...
+template <typename XprType, int BlockRows, int BlockCols, bool InnerPanel>
+class BlockImpl<XprType, BlockRows, BlockCols, InnerPanel, Dense>
+    : public internal::BlockImpl_dense<XprType, BlockRows, BlockCols, InnerPanel> {
+  typedef internal::BlockImpl_dense<XprType, BlockRows, BlockCols, InnerPanel> Impl;
+  typedef typename XprType::StorageIndex StorageIndex;
+
+ public:
+  typedef Impl Base;
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(BlockImpl)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl(XprType& xpr, Index i) : Impl(xpr, i) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl(XprType& xpr, Index startRow, Index startCol)
+      : Impl(xpr, startRow, startCol) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl(XprType& xpr, Index startRow, Index startCol, Index blockRows,
+                                                  Index blockCols)
+      : Impl(xpr, startRow, startCol, blockRows, blockCols) {}
+};
+
+namespace internal {
+
+/** \internal Internal implementation of dense Blocks in the general case. */
+template <typename XprType, int BlockRows, int BlockCols, bool InnerPanel, bool HasDirectAccess>
+class BlockImpl_dense : public internal::dense_xpr_base<Block<XprType, BlockRows, BlockCols, InnerPanel>>::type {
+  typedef Block<XprType, BlockRows, BlockCols, InnerPanel> BlockType;
+  typedef typename internal::ref_selector<XprType>::non_const_type XprTypeNested;
+
+ public:
+  typedef typename internal::dense_xpr_base<BlockType>::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(BlockType)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(BlockImpl_dense)
+
+  // class InnerIterator; // FIXME apparently never used
+
+  /** Column or Row constructor
+   */
+  EIGEN_DEVICE_FUNC inline BlockImpl_dense(XprType& xpr, Index i)
+      : m_xpr(xpr),
+        // It is a row if and only if BlockRows==1 and BlockCols==XprType::ColsAtCompileTime,
+        // and it is a column if and only if BlockRows==XprType::RowsAtCompileTime and BlockCols==1,
+        // all other cases are invalid.
+        // The case a 1x1 matrix seems ambiguous, but the result is the same anyway.
+        m_startRow((BlockRows == 1) && (BlockCols == XprType::ColsAtCompileTime) ? i : 0),
+        m_startCol((BlockRows == XprType::RowsAtCompileTime) && (BlockCols == 1) ? i : 0),
+        m_blockRows(BlockRows == 1 ? 1 : xpr.rows()),
+        m_blockCols(BlockCols == 1 ? 1 : xpr.cols()) {}
+
+  /** Fixed-size constructor
+   */
+  EIGEN_DEVICE_FUNC inline BlockImpl_dense(XprType& xpr, Index startRow, Index startCol)
+      : m_xpr(xpr), m_startRow(startRow), m_startCol(startCol), m_blockRows(BlockRows), m_blockCols(BlockCols) {}
+
+  /** Dynamic-size constructor
+   */
+  EIGEN_DEVICE_FUNC inline BlockImpl_dense(XprType& xpr, Index startRow, Index startCol, Index blockRows,
+                                           Index blockCols)
+      : m_xpr(xpr), m_startRow(startRow), m_startCol(startCol), m_blockRows(blockRows), m_blockCols(blockCols) {}
+
+  EIGEN_DEVICE_FUNC inline Index rows() const { return m_blockRows.value(); }
+  EIGEN_DEVICE_FUNC inline Index cols() const { return m_blockCols.value(); }
+
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index rowId, Index colId) {
+    EIGEN_STATIC_ASSERT_LVALUE(XprType)
+    return m_xpr.coeffRef(rowId + m_startRow.value(), colId + m_startCol.value());
+  }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const {
+    return m_xpr.derived().coeffRef(rowId + m_startRow.value(), colId + m_startCol.value());
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CoeffReturnType coeff(Index rowId, Index colId) const {
+    return m_xpr.coeff(rowId + m_startRow.value(), colId + m_startCol.value());
+  }
+
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index index) {
+    EIGEN_STATIC_ASSERT_LVALUE(XprType)
+    return m_xpr.coeffRef(m_startRow.value() + (RowsAtCompileTime == 1 ? 0 : index),
+                          m_startCol.value() + (RowsAtCompileTime == 1 ? index : 0));
+  }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const {
+    return m_xpr.coeffRef(m_startRow.value() + (RowsAtCompileTime == 1 ? 0 : index),
+                          m_startCol.value() + (RowsAtCompileTime == 1 ? index : 0));
+  }
+
+  EIGEN_DEVICE_FUNC inline const CoeffReturnType coeff(Index index) const {
+    return m_xpr.coeff(m_startRow.value() + (RowsAtCompileTime == 1 ? 0 : index),
+                       m_startCol.value() + (RowsAtCompileTime == 1 ? index : 0));
+  }
+
+  template <int LoadMode>
+  EIGEN_DEVICE_FUNC inline PacketScalar packet(Index rowId, Index colId) const {
+    return m_xpr.template packet<Unaligned>(rowId + m_startRow.value(), colId + m_startCol.value());
+  }
+
+  template <int LoadMode>
+  EIGEN_DEVICE_FUNC inline void writePacket(Index rowId, Index colId, const PacketScalar& val) {
+    m_xpr.template writePacket<Unaligned>(rowId + m_startRow.value(), colId + m_startCol.value(), val);
+  }
+
+  template <int LoadMode>
+  EIGEN_DEVICE_FUNC inline PacketScalar packet(Index index) const {
+    return m_xpr.template packet<Unaligned>(m_startRow.value() + (RowsAtCompileTime == 1 ? 0 : index),
+                                            m_startCol.value() + (RowsAtCompileTime == 1 ? index : 0));
+  }
+
+  template <int LoadMode>
+  EIGEN_DEVICE_FUNC inline void writePacket(Index index, const PacketScalar& val) {
+    m_xpr.template writePacket<Unaligned>(m_startRow.value() + (RowsAtCompileTime == 1 ? 0 : index),
+                                          m_startCol.value() + (RowsAtCompileTime == 1 ? index : 0), val);
+  }
+
+#ifdef EIGEN_PARSED_BY_DOXYGEN
+  /** \sa MapBase::data() */
+  EIGEN_DEVICE_FUNC constexpr const Scalar* data() const;
+  EIGEN_DEVICE_FUNC inline Index innerStride() const;
+  EIGEN_DEVICE_FUNC inline Index outerStride() const;
+#endif
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const internal::remove_all_t<XprTypeNested>& nestedExpression() const {
+    return m_xpr;
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE XprType& nestedExpression() { return m_xpr; }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR StorageIndex startRow() const EIGEN_NOEXCEPT {
+    return m_startRow.value();
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR StorageIndex startCol() const EIGEN_NOEXCEPT {
+    return m_startCol.value();
+  }
+
+ protected:
+  XprTypeNested m_xpr;
+  const internal::variable_if_dynamic<StorageIndex, (XprType::RowsAtCompileTime == 1 && BlockRows == 1) ? 0 : Dynamic>
+      m_startRow;
+  const internal::variable_if_dynamic<StorageIndex, (XprType::ColsAtCompileTime == 1 && BlockCols == 1) ? 0 : Dynamic>
+      m_startCol;
+  const internal::variable_if_dynamic<StorageIndex, RowsAtCompileTime> m_blockRows;
+  const internal::variable_if_dynamic<StorageIndex, ColsAtCompileTime> m_blockCols;
+};
+
+/** \internal Internal implementation of dense Blocks in the direct access case.*/
+template <typename XprType, int BlockRows, int BlockCols, bool InnerPanel>
+class BlockImpl_dense<XprType, BlockRows, BlockCols, InnerPanel, true>
+    : public MapBase<Block<XprType, BlockRows, BlockCols, InnerPanel>> {
+  typedef Block<XprType, BlockRows, BlockCols, InnerPanel> BlockType;
+  typedef typename internal::ref_selector<XprType>::non_const_type XprTypeNested;
+  enum { XprTypeIsRowMajor = (int(traits<XprType>::Flags) & RowMajorBit) != 0 };
+
+  /** \internal Returns base+offset (unless base is null, in which case returns null).
+   * Adding an offset to nullptr is undefined behavior, so we must avoid it.
+   */
+  template <typename Scalar>
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR EIGEN_ALWAYS_INLINE static Scalar* add_to_nullable_pointer(Scalar* base,
+                                                                                               Index offset) {
+    return base != nullptr ? base + offset : nullptr;
+  }
+
+ public:
+  typedef MapBase<BlockType> Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(BlockType)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(BlockImpl_dense)
+
+  /** Column or Row constructor
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl_dense(XprType& xpr, Index i)
+      : Base((BlockRows == 0 || BlockCols == 0)
+                 ? nullptr
+                 : add_to_nullable_pointer(
+                       xpr.data(),
+                       i * (((BlockRows == 1) && (BlockCols == XprType::ColsAtCompileTime) && (!XprTypeIsRowMajor)) ||
+                                    ((BlockRows == XprType::RowsAtCompileTime) && (BlockCols == 1) &&
+                                     (XprTypeIsRowMajor))
+                                ? xpr.innerStride()
+                                : xpr.outerStride())),
+             BlockRows == 1 ? 1 : xpr.rows(), BlockCols == 1 ? 1 : xpr.cols()),
+        m_xpr(xpr),
+        m_startRow((BlockRows == 1) && (BlockCols == XprType::ColsAtCompileTime) ? i : 0),
+        m_startCol((BlockRows == XprType::RowsAtCompileTime) && (BlockCols == 1) ? i : 0) {
+    init();
+  }
+
+  /** Fixed-size constructor
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl_dense(XprType& xpr, Index startRow, Index startCol)
+      : Base((BlockRows == 0 || BlockCols == 0)
+                 ? nullptr
+                 : add_to_nullable_pointer(xpr.data(),
+                                           xpr.innerStride() * (XprTypeIsRowMajor ? startCol : startRow) +
+                                               xpr.outerStride() * (XprTypeIsRowMajor ? startRow : startCol))),
+        m_xpr(xpr),
+        m_startRow(startRow),
+        m_startCol(startCol) {
+    init();
+  }
+
+  /** Dynamic-size constructor
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl_dense(XprType& xpr, Index startRow, Index startCol, Index blockRows,
+                                                        Index blockCols)
+      : Base((blockRows == 0 || blockCols == 0)
+                 ? nullptr
+                 : add_to_nullable_pointer(xpr.data(),
+                                           xpr.innerStride() * (XprTypeIsRowMajor ? startCol : startRow) +
+                                               xpr.outerStride() * (XprTypeIsRowMajor ? startRow : startCol)),
+             blockRows, blockCols),
+        m_xpr(xpr),
+        m_startRow(startRow),
+        m_startCol(startCol) {
+    init();
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const internal::remove_all_t<XprTypeNested>& nestedExpression() const
+      EIGEN_NOEXCEPT {
+    return m_xpr;
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE XprType& nestedExpression() { return m_xpr; }
+
+  /** \sa MapBase::innerStride() */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index innerStride() const EIGEN_NOEXCEPT {
+    return internal::traits<BlockType>::HasSameStorageOrderAsXprType ? m_xpr.innerStride() : m_xpr.outerStride();
+  }
+
+  /** \sa MapBase::outerStride() */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index outerStride() const EIGEN_NOEXCEPT {
+    return internal::traits<BlockType>::HasSameStorageOrderAsXprType ? m_xpr.outerStride() : m_xpr.innerStride();
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR StorageIndex startRow() const EIGEN_NOEXCEPT {
+    return m_startRow.value();
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR StorageIndex startCol() const EIGEN_NOEXCEPT {
+    return m_startCol.value();
+  }
+
+#ifndef __SUNPRO_CC
+  // FIXME sunstudio is not friendly with the above friend...
+  // META-FIXME there is no 'friend' keyword around here. Is this obsolete?
+ protected:
+#endif
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** \internal used by allowAligned() */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockImpl_dense(XprType& xpr, const Scalar* data, Index blockRows,
+                                                        Index blockCols)
+      : Base(data, blockRows, blockCols), m_xpr(xpr) {
+    init();
+  }
+#endif
+
+ protected:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void init() {
+    m_outerStride =
+        internal::traits<BlockType>::HasSameStorageOrderAsXprType ? m_xpr.outerStride() : m_xpr.innerStride();
+  }
+
+  XprTypeNested m_xpr;
+  const internal::variable_if_dynamic<StorageIndex, (XprType::RowsAtCompileTime == 1 && BlockRows == 1) ? 0 : Dynamic>
+      m_startRow;
+  const internal::variable_if_dynamic<StorageIndex, (XprType::ColsAtCompileTime == 1 && BlockCols == 1) ? 0 : Dynamic>
+      m_startCol;
+  Index m_outerStride;
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_BLOCK_H

include/Eigen/src/Core/CommaInitializer.h

@@ -0,0 +1,149 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_COMMAINITIALIZER_H
+#define EIGEN_COMMAINITIALIZER_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class CommaInitializer
+ * \ingroup Core_Module
+ *
+ * \brief Helper class used by the comma initializer operator
+ *
+ * This class is internally used to implement the comma initializer feature. It is
+ * the return type of MatrixBase::operator<<, and most of the time this is the only
+ * way it is used.
+ *
+ * \sa \blank \ref MatrixBaseCommaInitRef "MatrixBase::operator<<", CommaInitializer::finished()
+ */
+template <typename XprType>
+struct CommaInitializer {
+  typedef typename XprType::Scalar Scalar;
+
+  EIGEN_DEVICE_FUNC inline CommaInitializer(XprType& xpr, const Scalar& s)
+      : m_xpr(xpr), m_row(0), m_col(1), m_currentBlockRows(1) {
+    eigen_assert(m_xpr.rows() > 0 && m_xpr.cols() > 0 && "Cannot comma-initialize a 0x0 matrix (operator<<)");
+    m_xpr.coeffRef(0, 0) = s;
+  }
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline CommaInitializer(XprType& xpr, const DenseBase<OtherDerived>& other)
+      : m_xpr(xpr), m_row(0), m_col(other.cols()), m_currentBlockRows(other.rows()) {
+    eigen_assert(m_xpr.rows() >= other.rows() && m_xpr.cols() >= other.cols() &&
+                 "Cannot comma-initialize a 0x0 matrix (operator<<)");
+    m_xpr.template block<OtherDerived::RowsAtCompileTime, OtherDerived::ColsAtCompileTime>(0, 0, other.rows(),
+                                                                                           other.cols()) = other;
+  }
+
+  /* Copy/Move constructor which transfers ownership. This is crucial in
+   * absence of return value optimization to avoid assertions during destruction. */
+  // FIXME in C++11 mode this could be replaced by a proper RValue constructor
+  EIGEN_DEVICE_FUNC inline CommaInitializer(const CommaInitializer& o)
+      : m_xpr(o.m_xpr), m_row(o.m_row), m_col(o.m_col), m_currentBlockRows(o.m_currentBlockRows) {
+    // Mark original object as finished. In absence of R-value references we need to const_cast:
+    const_cast<CommaInitializer&>(o).m_row = m_xpr.rows();
+    const_cast<CommaInitializer&>(o).m_col = m_xpr.cols();
+    const_cast<CommaInitializer&>(o).m_currentBlockRows = 0;
+  }
+
+  /* inserts a scalar value in the target matrix */
+  EIGEN_DEVICE_FUNC CommaInitializer &operator,(const Scalar& s) {
+    if (m_col == m_xpr.cols()) {
+      m_row += m_currentBlockRows;
+      m_col = 0;
+      m_currentBlockRows = 1;
+      eigen_assert(m_row < m_xpr.rows() && "Too many rows passed to comma initializer (operator<<)");
+    }
+    eigen_assert(m_col < m_xpr.cols() && "Too many coefficients passed to comma initializer (operator<<)");
+    eigen_assert(m_currentBlockRows == 1);
+    m_xpr.coeffRef(m_row, m_col++) = s;
+    return *this;
+  }
+
+  /* inserts a matrix expression in the target matrix */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC CommaInitializer &operator,(const DenseBase<OtherDerived>& other) {
+    if (m_col == m_xpr.cols() && (other.cols() != 0 || other.rows() != m_currentBlockRows)) {
+      m_row += m_currentBlockRows;
+      m_col = 0;
+      m_currentBlockRows = other.rows();
+      eigen_assert(m_row + m_currentBlockRows <= m_xpr.rows() &&
+                   "Too many rows passed to comma initializer (operator<<)");
+    }
+    eigen_assert((m_col + other.cols() <= m_xpr.cols()) &&
+                 "Too many coefficients passed to comma initializer (operator<<)");
+    eigen_assert(m_currentBlockRows == other.rows());
+    m_xpr.template block<OtherDerived::RowsAtCompileTime, OtherDerived::ColsAtCompileTime>(m_row, m_col, other.rows(),
+                                                                                           other.cols()) = other;
+    m_col += other.cols();
+    return *this;
+  }
+
+  EIGEN_DEVICE_FUNC inline ~CommaInitializer()
+#if defined VERIFY_RAISES_ASSERT && (!defined EIGEN_NO_ASSERTION_CHECKING) && defined EIGEN_EXCEPTIONS
+      EIGEN_EXCEPTION_SPEC(Eigen::eigen_assert_exception)
+#endif
+  {
+    finished();
+  }
+
+  /** \returns the built matrix once all its coefficients have been set.
+   * Calling finished is 100% optional. Its purpose is to write expressions
+   * like this:
+   * \code
+   * quaternion.fromRotationMatrix((Matrix3f() << axis0, axis1, axis2).finished());
+   * \endcode
+   */
+  EIGEN_DEVICE_FUNC inline XprType& finished() {
+    eigen_assert(((m_row + m_currentBlockRows) == m_xpr.rows() || m_xpr.cols() == 0) && m_col == m_xpr.cols() &&
+                 "Too few coefficients passed to comma initializer (operator<<)");
+    return m_xpr;
+  }
+
+  XprType& m_xpr;            // target expression
+  Index m_row;               // current row id
+  Index m_col;               // current col id
+  Index m_currentBlockRows;  // current block height
+};
+
+/** \anchor MatrixBaseCommaInitRef
+ * Convenient operator to set the coefficients of a matrix.
+ *
+ * The coefficients must be provided in a row major order and exactly match
+ * the size of the matrix. Otherwise an assertion is raised.
+ *
+ * Example: \include MatrixBase_set.cpp
+ * Output: \verbinclude MatrixBase_set.out
+ *
+ * \note According the c++ standard, the argument expressions of this comma initializer are evaluated in arbitrary
+ * order.
+ *
+ * \sa CommaInitializer::finished(), class CommaInitializer
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline CommaInitializer<Derived> DenseBase<Derived>::operator<<(const Scalar& s) {
+  return CommaInitializer<Derived>(*static_cast<Derived*>(this), s);
+}
+
+/** \sa operator<<(const Scalar&) */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC inline CommaInitializer<Derived> DenseBase<Derived>::operator<<(
+    const DenseBase<OtherDerived>& other) {
+  return CommaInitializer<Derived>(*static_cast<Derived*>(this), other);
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_COMMAINITIALIZER_H

include/Eigen/src/Core/ConditionEstimator.h

@@ -0,0 +1,173 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2016 Rasmus Munk Larsen (rmlarsen@google.com)
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_CONDITIONESTIMATOR_H
+#define EIGEN_CONDITIONESTIMATOR_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Vector, typename RealVector, bool IsComplex>
+struct rcond_compute_sign {
+  static inline Vector run(const Vector& v) {
+    const RealVector v_abs = v.cwiseAbs();
+    return (v_abs.array() == static_cast<typename Vector::RealScalar>(0))
+        .select(Vector::Ones(v.size()), v.cwiseQuotient(v_abs));
+  }
+};
+
+// Partial specialization to avoid elementwise division for real vectors.
+template <typename Vector>
+struct rcond_compute_sign<Vector, Vector, false> {
+  static inline Vector run(const Vector& v) {
+    return (v.array() < static_cast<typename Vector::RealScalar>(0))
+        .select(-Vector::Ones(v.size()), Vector::Ones(v.size()));
+  }
+};
+
+/**
+ * \returns an estimate of ||inv(matrix)||_1 given a decomposition of
+ * \a matrix that implements .solve() and .adjoint().solve() methods.
+ *
+ * This function implements Algorithms 4.1 and 5.1 from
+ *   http://www.maths.manchester.ac.uk/~higham/narep/narep135.pdf
+ * which also forms the basis for the condition number estimators in
+ * LAPACK. Since at most 10 calls to the solve method of dec are
+ * performed, the total cost is O(dims^2), as opposed to O(dims^3)
+ * needed to compute the inverse matrix explicitly.
+ *
+ * The most common usage is in estimating the condition number
+ * ||matrix||_1 * ||inv(matrix)||_1. The first term ||matrix||_1 can be
+ * computed directly in O(n^2) operations.
+ *
+ * Supports the following decompositions: FullPivLU, PartialPivLU, LDLT, and
+ * LLT.
+ *
+ * \sa FullPivLU, PartialPivLU, LDLT, LLT.
+ */
+template <typename Decomposition>
+typename Decomposition::RealScalar rcond_invmatrix_L1_norm_estimate(const Decomposition& dec) {
+  typedef typename Decomposition::MatrixType MatrixType;
+  typedef typename Decomposition::Scalar Scalar;
+  typedef typename Decomposition::RealScalar RealScalar;
+  typedef typename internal::plain_col_type<MatrixType>::type Vector;
+  typedef typename internal::plain_col_type<MatrixType, RealScalar>::type RealVector;
+  const bool is_complex = (NumTraits<Scalar>::IsComplex != 0);
+
+  eigen_assert(dec.rows() == dec.cols());
+  const Index n = dec.rows();
+  if (n == 0) return 0;
+
+    // Disable Index to float conversion warning
+#ifdef __INTEL_COMPILER
+#pragma warning push
+#pragma warning(disable : 2259)
+#endif
+  Vector v = dec.solve(Vector::Ones(n) / Scalar(n));
+#ifdef __INTEL_COMPILER
+#pragma warning pop
+#endif
+
+  // lower_bound is a lower bound on
+  //   ||inv(matrix)||_1  = sup_v ||inv(matrix) v||_1 / ||v||_1
+  // and is the objective maximized by the ("super-") gradient ascent
+  // algorithm below.
+  RealScalar lower_bound = v.template lpNorm<1>();
+  if (n == 1) return lower_bound;
+
+  // Gradient ascent algorithm follows: We know that the optimum is achieved at
+  // one of the simplices v = e_i, so in each iteration we follow a
+  // super-gradient to move towards the optimal one.
+  RealScalar old_lower_bound = lower_bound;
+  Vector sign_vector(n);
+  Vector old_sign_vector;
+  Index v_max_abs_index = -1;
+  Index old_v_max_abs_index = v_max_abs_index;
+  for (int k = 0; k < 4; ++k) {
+    sign_vector = internal::rcond_compute_sign<Vector, RealVector, is_complex>::run(v);
+    if (k > 0 && !is_complex && sign_vector == old_sign_vector) {
+      // Break if the solution stagnated.
+      break;
+    }
+    // v_max_abs_index = argmax |real( inv(matrix)^T * sign_vector )|
+    v = dec.adjoint().solve(sign_vector);
+    v.real().cwiseAbs().maxCoeff(&v_max_abs_index);
+    if (v_max_abs_index == old_v_max_abs_index) {
+      // Break if the solution stagnated.
+      break;
+    }
+    // Move to the new simplex e_j, where j = v_max_abs_index.
+    v = dec.solve(Vector::Unit(n, v_max_abs_index));  // v = inv(matrix) * e_j.
+    lower_bound = v.template lpNorm<1>();
+    if (lower_bound <= old_lower_bound) {
+      // Break if the gradient step did not increase the lower_bound.
+      break;
+    }
+    if (!is_complex) {
+      old_sign_vector = sign_vector;
+    }
+    old_v_max_abs_index = v_max_abs_index;
+    old_lower_bound = lower_bound;
+  }
+  // The following calculates an independent estimate of ||matrix||_1 by
+  // multiplying matrix by a vector with entries of slowly increasing
+  // magnitude and alternating sign:
+  //   v_i = (-1)^{i} (1 + (i / (dim-1))), i = 0,...,dim-1.
+  // This improvement to Hager's algorithm above is due to Higham. It was
+  // added to make the algorithm more robust in certain corner cases where
+  // large elements in the matrix might otherwise escape detection due to
+  // exact cancellation (especially when op and op_adjoint correspond to a
+  // sequence of backsubstitutions and permutations), which could cause
+  // Hager's algorithm to vastly underestimate ||matrix||_1.
+  Scalar alternating_sign(RealScalar(1));
+  for (Index i = 0; i < n; ++i) {
+    // The static_cast is needed when Scalar is a complex and RealScalar implements expression templates
+    v[i] = alternating_sign * static_cast<RealScalar>(RealScalar(1) + (RealScalar(i) / (RealScalar(n - 1))));
+    alternating_sign = -alternating_sign;
+  }
+  v = dec.solve(v);
+  const RealScalar alternate_lower_bound = (2 * v.template lpNorm<1>()) / (3 * RealScalar(n));
+  return numext::maxi(lower_bound, alternate_lower_bound);
+}
+
+/** \brief Reciprocal condition number estimator.
+ *
+ * Computing a decomposition of a dense matrix takes O(n^3) operations, while
+ * this method estimates the condition number quickly and reliably in O(n^2)
+ * operations.
+ *
+ * \returns an estimate of the reciprocal condition number
+ * (1 / (||matrix||_1 * ||inv(matrix)||_1)) of matrix, given ||matrix||_1 and
+ * its decomposition. Supports the following decompositions: FullPivLU,
+ * PartialPivLU, LDLT, and LLT.
+ *
+ * \sa FullPivLU, PartialPivLU, LDLT, LLT.
+ */
+template <typename Decomposition>
+typename Decomposition::RealScalar rcond_estimate_helper(typename Decomposition::RealScalar matrix_norm,
+                                                         const Decomposition& dec) {
+  typedef typename Decomposition::RealScalar RealScalar;
+  eigen_assert(dec.rows() == dec.cols());
+  if (dec.rows() == 0) return NumTraits<RealScalar>::infinity();
+  if (numext::is_exactly_zero(matrix_norm)) return RealScalar(0);
+  if (dec.rows() == 1) return RealScalar(1);
+  const RealScalar inverse_matrix_norm = rcond_invmatrix_L1_norm_estimate(dec);
+  return (numext::is_exactly_zero(inverse_matrix_norm) ? RealScalar(0)
+                                                       : (RealScalar(1) / inverse_matrix_norm) / matrix_norm);
+}
+
+}  // namespace internal
+
+}  // namespace Eigen
+
+#endif

include/Eigen/src/Core/CoreIterators.h

@@ -0,0 +1,141 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_COREITERATORS_H
+#define EIGEN_COREITERATORS_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/* This file contains the respective InnerIterator definition of the expressions defined in Eigen/Core
+ */
+
+namespace internal {
+
+template <typename XprType, typename EvaluatorKind>
+class inner_iterator_selector;
+
+}
+
+/** \class InnerIterator
+ * \brief An InnerIterator allows to loop over the element of any matrix expression.
+ *
+ * \warning To be used with care because an evaluator is constructed every time an InnerIterator iterator is
+ * constructed.
+ *
+ * TODO: add a usage example
+ */
+template <typename XprType>
+class InnerIterator {
+ protected:
+  typedef internal::inner_iterator_selector<XprType, typename internal::evaluator_traits<XprType>::Kind> IteratorType;
+  typedef internal::evaluator<XprType> EvaluatorType;
+  typedef typename internal::traits<XprType>::Scalar Scalar;
+
+ public:
+  /** Construct an iterator over the \a outerId -th row or column of \a xpr */
+  InnerIterator(const XprType &xpr, const Index &outerId) : m_eval(xpr), m_iter(m_eval, outerId, xpr.innerSize()) {}
+
+  /// \returns the value of the current coefficient.
+  EIGEN_STRONG_INLINE Scalar value() const { return m_iter.value(); }
+  /** Increment the iterator \c *this to the next non-zero coefficient.
+   * Explicit zeros are not skipped over. To skip explicit zeros, see class SparseView
+   */
+  EIGEN_STRONG_INLINE InnerIterator &operator++() {
+    m_iter.operator++();
+    return *this;
+  }
+  EIGEN_STRONG_INLINE InnerIterator &operator+=(Index i) {
+    m_iter.operator+=(i);
+    return *this;
+  }
+  EIGEN_STRONG_INLINE InnerIterator operator+(Index i) {
+    InnerIterator result(*this);
+    result += i;
+    return result;
+  }
+
+  /// \returns the column or row index of the current coefficient.
+  EIGEN_STRONG_INLINE Index index() const { return m_iter.index(); }
+  /// \returns the row index of the current coefficient.
+  EIGEN_STRONG_INLINE Index row() const { return m_iter.row(); }
+  /// \returns the column index of the current coefficient.
+  EIGEN_STRONG_INLINE Index col() const { return m_iter.col(); }
+  /// \returns \c true if the iterator \c *this still references a valid coefficient.
+  EIGEN_STRONG_INLINE operator bool() const { return m_iter; }
+
+ protected:
+  EvaluatorType m_eval;
+  IteratorType m_iter;
+
+ private:
+  // If you get here, then you're not using the right InnerIterator type, e.g.:
+  //   SparseMatrix<double,RowMajor> A;
+  //   SparseMatrix<double>::InnerIterator it(A,0);
+  template <typename T>
+  InnerIterator(const EigenBase<T> &, Index outer);
+};
+
+namespace internal {
+
+// Generic inner iterator implementation for dense objects
+template <typename XprType>
+class inner_iterator_selector<XprType, IndexBased> {
+ protected:
+  typedef evaluator<XprType> EvaluatorType;
+  typedef typename traits<XprType>::Scalar Scalar;
+  enum { IsRowMajor = (XprType::Flags & RowMajorBit) == RowMajorBit };
+
+ public:
+  EIGEN_STRONG_INLINE inner_iterator_selector(const EvaluatorType &eval, const Index &outerId, const Index &innerSize)
+      : m_eval(eval), m_inner(0), m_outer(outerId), m_end(innerSize) {}
+
+  EIGEN_STRONG_INLINE Scalar value() const {
+    return (IsRowMajor) ? m_eval.coeff(m_outer, m_inner) : m_eval.coeff(m_inner, m_outer);
+  }
+
+  EIGEN_STRONG_INLINE inner_iterator_selector &operator++() {
+    m_inner++;
+    return *this;
+  }
+
+  EIGEN_STRONG_INLINE Index index() const { return m_inner; }
+  inline Index row() const { return IsRowMajor ? m_outer : index(); }
+  inline Index col() const { return IsRowMajor ? index() : m_outer; }
+
+  EIGEN_STRONG_INLINE operator bool() const { return m_inner < m_end && m_inner >= 0; }
+
+ protected:
+  const EvaluatorType &m_eval;
+  Index m_inner;
+  const Index m_outer;
+  const Index m_end;
+};
+
+// For iterator-based evaluator, inner-iterator is already implemented as
+// evaluator<>::InnerIterator
+template <typename XprType>
+class inner_iterator_selector<XprType, IteratorBased> : public evaluator<XprType>::InnerIterator {
+ protected:
+  typedef typename evaluator<XprType>::InnerIterator Base;
+  typedef evaluator<XprType> EvaluatorType;
+
+ public:
+  EIGEN_STRONG_INLINE inner_iterator_selector(const EvaluatorType &eval, const Index &outerId,
+                                              const Index & /*innerSize*/)
+      : Base(eval, outerId) {}
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_COREITERATORS_H

include/Eigen/src/Core/CwiseBinaryOp.h

@@ -0,0 +1,166 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_CWISE_BINARY_OP_H
+#define EIGEN_CWISE_BINARY_OP_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename BinaryOp, typename Lhs, typename Rhs>
+struct traits<CwiseBinaryOp<BinaryOp, Lhs, Rhs>> {
+  // we must not inherit from traits<Lhs> since it has
+  // the potential to cause problems with MSVC
+  typedef remove_all_t<Lhs> Ancestor;
+  typedef typename traits<Ancestor>::XprKind XprKind;
+  enum {
+    RowsAtCompileTime = traits<Ancestor>::RowsAtCompileTime,
+    ColsAtCompileTime = traits<Ancestor>::ColsAtCompileTime,
+    MaxRowsAtCompileTime = traits<Ancestor>::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = traits<Ancestor>::MaxColsAtCompileTime
+  };
+
+  // even though we require Lhs and Rhs to have the same scalar type (see CwiseBinaryOp constructor),
+  // we still want to handle the case when the result type is different.
+  typedef typename result_of<BinaryOp(const typename Lhs::Scalar&, const typename Rhs::Scalar&)>::type Scalar;
+  typedef typename cwise_promote_storage_type<typename traits<Lhs>::StorageKind, typename traits<Rhs>::StorageKind,
+                                              BinaryOp>::ret StorageKind;
+  typedef typename promote_index_type<typename traits<Lhs>::StorageIndex, typename traits<Rhs>::StorageIndex>::type
+      StorageIndex;
+  typedef typename Lhs::Nested LhsNested;
+  typedef typename Rhs::Nested RhsNested;
+  typedef std::remove_reference_t<LhsNested> LhsNested_;
+  typedef std::remove_reference_t<RhsNested> RhsNested_;
+  enum {
+    Flags = cwise_promote_storage_order<typename traits<Lhs>::StorageKind, typename traits<Rhs>::StorageKind,
+                                        LhsNested_::Flags & RowMajorBit, RhsNested_::Flags & RowMajorBit>::value
+  };
+};
+}  // end namespace internal
+
+template <typename BinaryOp, typename Lhs, typename Rhs, typename StorageKind>
+class CwiseBinaryOpImpl;
+
+/** \class CwiseBinaryOp
+ * \ingroup Core_Module
+ *
+ * \brief Generic expression where a coefficient-wise binary operator is applied to two expressions
+ *
+ * \tparam BinaryOp template functor implementing the operator
+ * \tparam LhsType the type of the left-hand side
+ * \tparam RhsType the type of the right-hand side
+ *
+ * This class represents an expression  where a coefficient-wise binary operator is applied to two expressions.
+ * It is the return type of binary operators, by which we mean only those binary operators where
+ * both the left-hand side and the right-hand side are Eigen expressions.
+ * For example, the return type of matrix1+matrix2 is a CwiseBinaryOp.
+ *
+ * Most of the time, this is the only way that it is used, so you typically don't have to name
+ * CwiseBinaryOp types explicitly.
+ *
+ * \sa MatrixBase::binaryExpr(const MatrixBase<OtherDerived> &,const CustomBinaryOp &) const, class CwiseUnaryOp, class
+ * CwiseNullaryOp
+ */
+template <typename BinaryOp, typename LhsType, typename RhsType>
+class CwiseBinaryOp : public CwiseBinaryOpImpl<BinaryOp, LhsType, RhsType,
+                                               typename internal::cwise_promote_storage_type<
+                                                   typename internal::traits<LhsType>::StorageKind,
+                                                   typename internal::traits<RhsType>::StorageKind, BinaryOp>::ret>,
+                      internal::no_assignment_operator {
+ public:
+  typedef internal::remove_all_t<BinaryOp> Functor;
+  typedef internal::remove_all_t<LhsType> Lhs;
+  typedef internal::remove_all_t<RhsType> Rhs;
+
+  typedef typename CwiseBinaryOpImpl<
+      BinaryOp, LhsType, RhsType,
+      typename internal::cwise_promote_storage_type<typename internal::traits<LhsType>::StorageKind,
+                                                    typename internal::traits<Rhs>::StorageKind, BinaryOp>::ret>::Base
+      Base;
+  EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseBinaryOp)
+
+  EIGEN_CHECK_BINARY_COMPATIBILIY(BinaryOp, typename Lhs::Scalar, typename Rhs::Scalar)
+  EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Lhs, Rhs)
+
+  typedef typename internal::ref_selector<LhsType>::type LhsNested;
+  typedef typename internal::ref_selector<RhsType>::type RhsNested;
+  typedef std::remove_reference_t<LhsNested> LhsNested_;
+  typedef std::remove_reference_t<RhsNested> RhsNested_;
+
+#if EIGEN_COMP_MSVC
+  // Required for Visual Studio or the Copy constructor will probably not get inlined!
+  EIGEN_STRONG_INLINE CwiseBinaryOp(const CwiseBinaryOp<BinaryOp, LhsType, RhsType>&) = default;
+#endif
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CwiseBinaryOp(const Lhs& aLhs, const Rhs& aRhs,
+                                                      const BinaryOp& func = BinaryOp())
+      : m_lhs(aLhs), m_rhs(aRhs), m_functor(func) {
+    eigen_assert(aLhs.rows() == aRhs.rows() && aLhs.cols() == aRhs.cols());
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT {
+    // return the fixed size type if available to enable compile time optimizations
+    return internal::traits<internal::remove_all_t<LhsNested>>::RowsAtCompileTime == Dynamic ? m_rhs.rows()
+                                                                                             : m_lhs.rows();
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT {
+    // return the fixed size type if available to enable compile time optimizations
+    return internal::traits<internal::remove_all_t<LhsNested>>::ColsAtCompileTime == Dynamic ? m_rhs.cols()
+                                                                                             : m_lhs.cols();
+  }
+
+  /** \returns the left hand side nested expression */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const LhsNested_& lhs() const { return m_lhs; }
+  /** \returns the right hand side nested expression */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const RhsNested_& rhs() const { return m_rhs; }
+  /** \returns the functor representing the binary operation */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const BinaryOp& functor() const { return m_functor; }
+
+ protected:
+  LhsNested m_lhs;
+  RhsNested m_rhs;
+  const BinaryOp m_functor;
+};
+
+// Generic API dispatcher
+template <typename BinaryOp, typename Lhs, typename Rhs, typename StorageKind>
+class CwiseBinaryOpImpl : public internal::generic_xpr_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs>>::type {
+ public:
+  typedef typename internal::generic_xpr_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs>>::type Base;
+};
+
+/** replaces \c *this by \c *this - \a other.
+ *
+ * \returns a reference to \c *this
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator-=(const MatrixBase<OtherDerived>& other) {
+  call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar, typename OtherDerived::Scalar>());
+  return derived();
+}
+
+/** replaces \c *this by \c *this + \a other.
+ *
+ * \returns a reference to \c *this
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator+=(const MatrixBase<OtherDerived>& other) {
+  call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar, typename OtherDerived::Scalar>());
+  return derived();
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_CWISE_BINARY_OP_H

include/Eigen/src/Core/CwiseNullaryOp.h

@@ -0,0 +1,977 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_CWISE_NULLARY_OP_H
+#define EIGEN_CWISE_NULLARY_OP_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename NullaryOp, typename PlainObjectType>
+struct traits<CwiseNullaryOp<NullaryOp, PlainObjectType> > : traits<PlainObjectType> {
+  enum { Flags = traits<PlainObjectType>::Flags & RowMajorBit };
+};
+
+}  // namespace internal
+
+/** \class CwiseNullaryOp
+  * \ingroup Core_Module
+  *
+  * \brief Generic expression of a matrix where all coefficients are defined by a functor
+  *
+  * \tparam NullaryOp template functor implementing the operator
+  * \tparam PlainObjectType the underlying plain matrix/array type
+  *
+  * This class represents an expression of a generic nullary operator.
+  * It is the return type of the Ones(), Zero(), Constant(), Identity() and Random() methods,
+  * and most of the time this is the only way it is used.
+  *
+  * However, if you want to write a function returning such an expression, you
+  * will need to use this class.
+  *
+  * The functor NullaryOp must expose one of the following method:
+    <table class="manual">
+    <tr            ><td>\c operator()() </td><td>if the procedural generation does not depend on the coefficient entries
+  (e.g., random numbers)</td></tr> <tr class="alt"><td>\c operator()(Index i)</td><td>if the procedural generation makes
+  sense for vectors only and that it depends on the coefficient index \c i (e.g., linspace) </td></tr> <tr ><td>\c
+  operator()(Index i,Index j)</td><td>if the procedural generation depends on the matrix coordinates \c i, \c j (e.g.,
+  to generate a checkerboard with 0 and 1)</td></tr>
+    </table>
+  * It is also possible to expose the last two operators if the generation makes sense for matrices but can be optimized
+  for vectors.
+  *
+  * See DenseBase::NullaryExpr(Index,const CustomNullaryOp&) for an example binding
+  * C++11 random number generators.
+  *
+  * A nullary expression can also be used to implement custom sophisticated matrix manipulations
+  * that cannot be covered by the existing set of natively supported matrix manipulations.
+  * See this \ref TopicCustomizing_NullaryExpr "page" for some examples and additional explanations
+  * on the behavior of CwiseNullaryOp.
+  *
+  * \sa class CwiseUnaryOp, class CwiseBinaryOp, DenseBase::NullaryExpr
+  */
+template <typename NullaryOp, typename PlainObjectType>
+class CwiseNullaryOp : public internal::dense_xpr_base<CwiseNullaryOp<NullaryOp, PlainObjectType> >::type,
+                       internal::no_assignment_operator {
+ public:
+  typedef typename internal::dense_xpr_base<CwiseNullaryOp>::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(CwiseNullaryOp)
+
+  EIGEN_DEVICE_FUNC CwiseNullaryOp(Index rows, Index cols, const NullaryOp& func = NullaryOp())
+      : m_rows(rows), m_cols(cols), m_functor(func) {
+    eigen_assert(rows >= 0 && (RowsAtCompileTime == Dynamic || RowsAtCompileTime == rows) && cols >= 0 &&
+                 (ColsAtCompileTime == Dynamic || ColsAtCompileTime == cols));
+  }
+  EIGEN_DEVICE_FUNC CwiseNullaryOp(Index size, const NullaryOp& func = NullaryOp())
+      : CwiseNullaryOp(RowsAtCompileTime == 1 ? 1 : size, RowsAtCompileTime == 1 ? size : 1, func) {
+    EIGEN_STATIC_ASSERT(CwiseNullaryOp::IsVectorAtCompileTime, YOU_TRIED_CALLING_A_VECTOR_METHOD_ON_A_MATRIX);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const { return m_rows.value(); }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const { return m_cols.value(); }
+
+  /** \returns the functor representing the nullary operation */
+  EIGEN_DEVICE_FUNC const NullaryOp& functor() const { return m_functor; }
+
+ protected:
+  const internal::variable_if_dynamic<Index, RowsAtCompileTime> m_rows;
+  const internal::variable_if_dynamic<Index, ColsAtCompileTime> m_cols;
+  const NullaryOp m_functor;
+};
+
+/** \returns an expression of a matrix defined by a custom functor \a func
+ *
+ * The parameters \a rows and \a cols are the number of rows and of columns of
+ * the returned matrix. Must be compatible with this MatrixBase type.
+ *
+ * This variant is meant to be used for dynamic-size matrix types. For fixed-size types,
+ * it is redundant to pass \a rows and \a cols as arguments, so Zero() should be used
+ * instead.
+ *
+ * The template parameter \a CustomNullaryOp is the type of the functor.
+ *
+ * \sa class CwiseNullaryOp
+ */
+template <typename Derived>
+template <typename CustomNullaryOp>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+    const CwiseNullaryOp<CustomNullaryOp, typename DenseBase<Derived>::PlainObject>
+#else
+    const CwiseNullaryOp<CustomNullaryOp, PlainObject>
+#endif
+    DenseBase<Derived>::NullaryExpr(Index rows, Index cols, const CustomNullaryOp& func) {
+  return CwiseNullaryOp<CustomNullaryOp, PlainObject>(rows, cols, func);
+}
+
+/** \returns an expression of a matrix defined by a custom functor \a func
+ *
+ * The parameter \a size is the size of the returned vector.
+ * Must be compatible with this MatrixBase type.
+ *
+ * \only_for_vectors
+ *
+ * This variant is meant to be used for dynamic-size vector types. For fixed-size types,
+ * it is redundant to pass \a size as argument, so Zero() should be used
+ * instead.
+ *
+ * The template parameter \a CustomNullaryOp is the type of the functor.
+ *
+ * Here is an example with C++11 random generators: \include random_cpp11.cpp
+ * Output: \verbinclude random_cpp11.out
+ *
+ * \sa class CwiseNullaryOp
+ */
+template <typename Derived>
+template <typename CustomNullaryOp>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+    const CwiseNullaryOp<CustomNullaryOp, typename DenseBase<Derived>::PlainObject>
+#else
+    const CwiseNullaryOp<CustomNullaryOp, PlainObject>
+#endif
+    DenseBase<Derived>::NullaryExpr(Index size, const CustomNullaryOp& func) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  if (RowsAtCompileTime == 1)
+    return CwiseNullaryOp<CustomNullaryOp, PlainObject>(1, size, func);
+  else
+    return CwiseNullaryOp<CustomNullaryOp, PlainObject>(size, 1, func);
+}
+
+/** \returns an expression of a matrix defined by a custom functor \a func
+ *
+ * This variant is only for fixed-size DenseBase types. For dynamic-size types, you
+ * need to use the variants taking size arguments.
+ *
+ * The template parameter \a CustomNullaryOp is the type of the functor.
+ *
+ * \sa class CwiseNullaryOp
+ */
+template <typename Derived>
+template <typename CustomNullaryOp>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+    const CwiseNullaryOp<CustomNullaryOp, typename DenseBase<Derived>::PlainObject>
+#else
+    const CwiseNullaryOp<CustomNullaryOp, PlainObject>
+#endif
+    DenseBase<Derived>::NullaryExpr(const CustomNullaryOp& func) {
+  return CwiseNullaryOp<CustomNullaryOp, PlainObject>(RowsAtCompileTime, ColsAtCompileTime, func);
+}
+
+/** \returns an expression of a constant matrix of value \a value
+ *
+ * The parameters \a rows and \a cols are the number of rows and of columns of
+ * the returned matrix. Must be compatible with this DenseBase type.
+ *
+ * This variant is meant to be used for dynamic-size matrix types. For fixed-size types,
+ * it is redundant to pass \a rows and \a cols as arguments, so Zero() should be used
+ * instead.
+ *
+ * The template parameter \a CustomNullaryOp is the type of the functor.
+ *
+ * \sa class CwiseNullaryOp
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType
+DenseBase<Derived>::Constant(Index rows, Index cols, const Scalar& value) {
+  return DenseBase<Derived>::NullaryExpr(rows, cols, internal::scalar_constant_op<Scalar>(value));
+}
+
+/** \returns an expression of a constant matrix of value \a value
+ *
+ * The parameter \a size is the size of the returned vector.
+ * Must be compatible with this DenseBase type.
+ *
+ * \only_for_vectors
+ *
+ * This variant is meant to be used for dynamic-size vector types. For fixed-size types,
+ * it is redundant to pass \a size as argument, so Zero() should be used
+ * instead.
+ *
+ * The template parameter \a CustomNullaryOp is the type of the functor.
+ *
+ * \sa class CwiseNullaryOp
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType
+DenseBase<Derived>::Constant(Index size, const Scalar& value) {
+  return DenseBase<Derived>::NullaryExpr(size, internal::scalar_constant_op<Scalar>(value));
+}
+
+/** \returns an expression of a constant matrix of value \a value
+ *
+ * This variant is only for fixed-size DenseBase types. For dynamic-size types, you
+ * need to use the variants taking size arguments.
+ *
+ * The template parameter \a CustomNullaryOp is the type of the functor.
+ *
+ * \sa class CwiseNullaryOp
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType
+DenseBase<Derived>::Constant(const Scalar& value) {
+  EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived)
+  return DenseBase<Derived>::NullaryExpr(RowsAtCompileTime, ColsAtCompileTime,
+                                         internal::scalar_constant_op<Scalar>(value));
+}
+
+/** \deprecated because of accuracy loss. In Eigen 3.3, it is an alias for LinSpaced(Index,const Scalar&,const Scalar&)
+ *
+ * \only_for_vectors
+ *
+ * Example: \include DenseBase_LinSpaced_seq_deprecated.cpp
+ * Output: \verbinclude DenseBase_LinSpaced_seq_deprecated.out
+ *
+ * \sa LinSpaced(Index,const Scalar&, const Scalar&), setLinSpaced(Index,const Scalar&,const Scalar&)
+ */
+template <typename Derived>
+EIGEN_DEPRECATED EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<
+    Derived>::RandomAccessLinSpacedReturnType
+DenseBase<Derived>::LinSpaced(Sequential_t, Index size, const Scalar& low, const Scalar& high) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  return DenseBase<Derived>::NullaryExpr(size, internal::linspaced_op<Scalar>(low, high, size));
+}
+
+/** \deprecated because of accuracy loss. In Eigen 3.3, it is an alias for LinSpaced(const Scalar&,const Scalar&)
+ *
+ * \sa LinSpaced(const Scalar&, const Scalar&)
+ */
+template <typename Derived>
+EIGEN_DEPRECATED EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<
+    Derived>::RandomAccessLinSpacedReturnType
+DenseBase<Derived>::LinSpaced(Sequential_t, const Scalar& low, const Scalar& high) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived)
+  return DenseBase<Derived>::NullaryExpr(Derived::SizeAtCompileTime,
+                                         internal::linspaced_op<Scalar>(low, high, Derived::SizeAtCompileTime));
+}
+
+/**
+ * \brief Sets a linearly spaced vector.
+ *
+ * The function generates 'size' equally spaced values in the closed interval [low,high].
+ * When size is set to 1, a vector of length 1 containing 'high' is returned.
+ *
+ * \only_for_vectors
+ *
+ * Example: \include DenseBase_LinSpaced.cpp
+ * Output: \verbinclude DenseBase_LinSpaced.out
+ *
+ * For integer scalar types, an even spacing is possible if and only if the length of the range,
+ * i.e., \c high-low is a scalar multiple of \c size-1, or if \c size is a scalar multiple of the
+ * number of values \c high-low+1 (meaning each value can be repeated the same number of time).
+ * If one of these two considions is not satisfied, then \c high is lowered to the largest value
+ * satisfying one of this constraint.
+ * Here are some examples:
+ *
+ * Example: \include DenseBase_LinSpacedInt.cpp
+ * Output: \verbinclude DenseBase_LinSpacedInt.out
+ *
+ * \sa setLinSpaced(Index,const Scalar&,const Scalar&), CwiseNullaryOp
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::RandomAccessLinSpacedReturnType
+DenseBase<Derived>::LinSpaced(Index size, const Scalar& low, const Scalar& high) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  return DenseBase<Derived>::NullaryExpr(size, internal::linspaced_op<Scalar>(low, high, size));
+}
+
+/**
+ * \copydoc DenseBase::LinSpaced(Index, const Scalar&, const Scalar&)
+ * Special version for fixed size types which does not require the size parameter.
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::RandomAccessLinSpacedReturnType
+DenseBase<Derived>::LinSpaced(const Scalar& low, const Scalar& high) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived)
+  return DenseBase<Derived>::NullaryExpr(Derived::SizeAtCompileTime,
+                                         internal::linspaced_op<Scalar>(low, high, Derived::SizeAtCompileTime));
+}
+
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::RandomAccessEqualSpacedReturnType
+DenseBase<Derived>::EqualSpaced(Index size, const Scalar& low, const Scalar& step) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  return DenseBase<Derived>::NullaryExpr(size, internal::equalspaced_op<Scalar>(low, step));
+}
+
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::RandomAccessEqualSpacedReturnType
+DenseBase<Derived>::EqualSpaced(const Scalar& low, const Scalar& step) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  return DenseBase<Derived>::NullaryExpr(Derived::SizeAtCompileTime, internal::equalspaced_op<Scalar>(low, step));
+}
+
+/** \returns true if all coefficients in this matrix are approximately equal to \a val, to within precision \a prec */
+template <typename Derived>
+EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isApproxToConstant(const Scalar& val, const RealScalar& prec) const {
+  typename internal::nested_eval<Derived, 1>::type self(derived());
+  for (Index j = 0; j < cols(); ++j)
+    for (Index i = 0; i < rows(); ++i)
+      if (!internal::isApprox(self.coeff(i, j), val, prec)) return false;
+  return true;
+}
+
+/** This is just an alias for isApproxToConstant().
+ *
+ * \returns true if all coefficients in this matrix are approximately equal to \a value, to within precision \a prec */
+template <typename Derived>
+EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isConstant(const Scalar& val, const RealScalar& prec) const {
+  return isApproxToConstant(val, prec);
+}
+
+/** Alias for setConstant(): sets all coefficients in this expression to \a val.
+ *
+ * \sa setConstant(), Constant(), class CwiseNullaryOp
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void DenseBase<Derived>::fill(const Scalar& val) {
+  setConstant(val);
+}
+
+/** Sets all coefficients in this expression to value \a val.
+ *
+ * \sa fill(), setConstant(Index,const Scalar&), setConstant(Index,Index,const Scalar&), setZero(), setOnes(),
+ * Constant(), class CwiseNullaryOp, setZero(), setOnes()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setConstant(const Scalar& val) {
+  internal::eigen_fill_impl<Derived>::run(derived(), val);
+  return derived();
+}
+
+/** Resizes to the given \a size, and sets all coefficients in this expression to the given value \a val.
+ *
+ * \only_for_vectors
+ *
+ * Example: \include Matrix_setConstant_int.cpp
+ * Output: \verbinclude Matrix_setConstant_int.out
+ *
+ * \sa MatrixBase::setConstant(const Scalar&), setConstant(Index,Index,const Scalar&), class CwiseNullaryOp,
+ * MatrixBase::Constant(const Scalar&)
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setConstant(Index size, const Scalar& val) {
+  resize(size);
+  return setConstant(val);
+}
+
+/** Resizes to the given size, and sets all coefficients in this expression to the given value \a val.
+ *
+ * \param rows the new number of rows
+ * \param cols the new number of columns
+ * \param val the value to which all coefficients are set
+ *
+ * Example: \include Matrix_setConstant_int_int.cpp
+ * Output: \verbinclude Matrix_setConstant_int_int.out
+ *
+ * \sa MatrixBase::setConstant(const Scalar&), setConstant(Index,const Scalar&), class CwiseNullaryOp,
+ * MatrixBase::Constant(const Scalar&)
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setConstant(Index rows, Index cols,
+                                                                                     const Scalar& val) {
+  resize(rows, cols);
+  return setConstant(val);
+}
+
+/** Resizes to the given size, changing only the number of columns, and sets all
+ * coefficients in this expression to the given value \a val. For the parameter
+ * of type NoChange_t, just pass the special value \c NoChange.
+ *
+ * \sa MatrixBase::setConstant(const Scalar&), setConstant(Index,const Scalar&), class CwiseNullaryOp,
+ * MatrixBase::Constant(const Scalar&)
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setConstant(NoChange_t, Index cols,
+                                                                                     const Scalar& val) {
+  return setConstant(rows(), cols, val);
+}
+
+/** Resizes to the given size, changing only the number of rows, and sets all
+ * coefficients in this expression to the given value \a val. For the parameter
+ * of type NoChange_t, just pass the special value \c NoChange.
+ *
+ * \sa MatrixBase::setConstant(const Scalar&), setConstant(Index,const Scalar&), class CwiseNullaryOp,
+ * MatrixBase::Constant(const Scalar&)
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setConstant(Index rows, NoChange_t,
+                                                                                     const Scalar& val) {
+  return setConstant(rows, cols(), val);
+}
+
+/**
+ * \brief Sets a linearly spaced vector.
+ *
+ * The function generates 'size' equally spaced values in the closed interval [low,high].
+ * When size is set to 1, a vector of length 1 containing 'high' is returned.
+ *
+ * \only_for_vectors
+ *
+ * Example: \include DenseBase_setLinSpaced.cpp
+ * Output: \verbinclude DenseBase_setLinSpaced.out
+ *
+ * For integer scalar types, do not miss the explanations on the definition
+ * of \link LinSpaced(Index,const Scalar&,const Scalar&) even spacing \endlink.
+ *
+ * \sa LinSpaced(Index,const Scalar&,const Scalar&), CwiseNullaryOp
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setLinSpaced(Index newSize, const Scalar& low,
+                                                                                const Scalar& high) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  return derived() = Derived::NullaryExpr(newSize, internal::linspaced_op<Scalar>(low, high, newSize));
+}
+
+/**
+ * \brief Sets a linearly spaced vector.
+ *
+ * The function fills \c *this with equally spaced values in the closed interval [low,high].
+ * When size is set to 1, a vector of length 1 containing 'high' is returned.
+ *
+ * \only_for_vectors
+ *
+ * For integer scalar types, do not miss the explanations on the definition
+ * of \link LinSpaced(Index,const Scalar&,const Scalar&) even spacing \endlink.
+ *
+ * \sa LinSpaced(Index,const Scalar&,const Scalar&), setLinSpaced(Index, const Scalar&, const Scalar&), CwiseNullaryOp
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setLinSpaced(const Scalar& low, const Scalar& high) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  return setLinSpaced(size(), low, high);
+}
+
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setEqualSpaced(Index newSize, const Scalar& low,
+                                                                                  const Scalar& step) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  return derived() = Derived::NullaryExpr(newSize, internal::equalspaced_op<Scalar>(low, step));
+}
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setEqualSpaced(const Scalar& low,
+                                                                                  const Scalar& step) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  return setEqualSpaced(size(), low, step);
+}
+
+// zero:
+
+/** \returns an expression of a zero matrix.
+ *
+ * The parameters \a rows and \a cols are the number of rows and of columns of
+ * the returned matrix. Must be compatible with this MatrixBase type.
+ *
+ * This variant is meant to be used for dynamic-size matrix types. For fixed-size types,
+ * it is redundant to pass \a rows and \a cols as arguments, so Zero() should be used
+ * instead.
+ *
+ * Example: \include MatrixBase_zero_int_int.cpp
+ * Output: \verbinclude MatrixBase_zero_int_int.out
+ *
+ * \sa Zero(), Zero(Index)
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ZeroReturnType DenseBase<Derived>::Zero(
+    Index rows, Index cols) {
+  return ZeroReturnType(rows, cols);
+}
+
+/** \returns an expression of a zero vector.
+ *
+ * The parameter \a size is the size of the returned vector.
+ * Must be compatible with this MatrixBase type.
+ *
+ * \only_for_vectors
+ *
+ * This variant is meant to be used for dynamic-size vector types. For fixed-size types,
+ * it is redundant to pass \a size as argument, so Zero() should be used
+ * instead.
+ *
+ * Example: \include MatrixBase_zero_int.cpp
+ * Output: \verbinclude MatrixBase_zero_int.out
+ *
+ * \sa Zero(), Zero(Index,Index)
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ZeroReturnType DenseBase<Derived>::Zero(
+    Index size) {
+  return ZeroReturnType(size);
+}
+
+/** \returns an expression of a fixed-size zero matrix or vector.
+ *
+ * This variant is only for fixed-size MatrixBase types. For dynamic-size types, you
+ * need to use the variants taking size arguments.
+ *
+ * Example: \include MatrixBase_zero.cpp
+ * Output: \verbinclude MatrixBase_zero.out
+ *
+ * \sa Zero(Index), Zero(Index,Index)
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ZeroReturnType DenseBase<Derived>::Zero() {
+  return ZeroReturnType(RowsAtCompileTime, ColsAtCompileTime);
+}
+
+/** \returns true if *this is approximately equal to the zero matrix,
+ *          within the precision given by \a prec.
+ *
+ * Example: \include MatrixBase_isZero.cpp
+ * Output: \verbinclude MatrixBase_isZero.out
+ *
+ * \sa class CwiseNullaryOp, Zero()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isZero(const RealScalar& prec) const {
+  typename internal::nested_eval<Derived, 1>::type self(derived());
+  for (Index j = 0; j < cols(); ++j)
+    for (Index i = 0; i < rows(); ++i)
+      if (!internal::isMuchSmallerThan(self.coeff(i, j), static_cast<Scalar>(1), prec)) return false;
+  return true;
+}
+
+/** Sets all coefficients in this expression to zero.
+ *
+ * Example: \include MatrixBase_setZero.cpp
+ * Output: \verbinclude MatrixBase_setZero.out
+ *
+ * \sa class CwiseNullaryOp, Zero()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setZero() {
+  internal::eigen_zero_impl<Derived>::run(derived());
+  return derived();
+}
+
+/** Resizes to the given \a size, and sets all coefficients in this expression to zero.
+ *
+ * \only_for_vectors
+ *
+ * Example: \include Matrix_setZero_int.cpp
+ * Output: \verbinclude Matrix_setZero_int.out
+ *
+ * \sa DenseBase::setZero(), setZero(Index,Index), class CwiseNullaryOp, DenseBase::Zero()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setZero(Index newSize) {
+  resize(newSize);
+  return setZero();
+}
+
+/** Resizes to the given size, and sets all coefficients in this expression to zero.
+ *
+ * \param rows the new number of rows
+ * \param cols the new number of columns
+ *
+ * Example: \include Matrix_setZero_int_int.cpp
+ * Output: \verbinclude Matrix_setZero_int_int.out
+ *
+ * \sa DenseBase::setZero(), setZero(Index), class CwiseNullaryOp, DenseBase::Zero()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setZero(Index rows, Index cols) {
+  resize(rows, cols);
+  return setZero();
+}
+
+/** Resizes to the given size, changing only the number of columns, and sets all
+ * coefficients in this expression to zero. For the parameter of type NoChange_t,
+ * just pass the special value \c NoChange.
+ *
+ * \sa DenseBase::setZero(), setZero(Index), setZero(Index, Index), setZero(Index, NoChange_t), class CwiseNullaryOp,
+ * DenseBase::Zero()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setZero(NoChange_t, Index cols) {
+  return setZero(rows(), cols);
+}
+
+/** Resizes to the given size, changing only the number of rows, and sets all
+ * coefficients in this expression to zero. For the parameter of type NoChange_t,
+ * just pass the special value \c NoChange.
+ *
+ * \sa DenseBase::setZero(), setZero(Index), setZero(Index, Index), setZero(NoChange_t, Index), class CwiseNullaryOp,
+ * DenseBase::Zero()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setZero(Index rows, NoChange_t) {
+  return setZero(rows, cols());
+}
+
+// ones:
+
+/** \returns an expression of a matrix where all coefficients equal one.
+ *
+ * The parameters \a rows and \a cols are the number of rows and of columns of
+ * the returned matrix. Must be compatible with this MatrixBase type.
+ *
+ * This variant is meant to be used for dynamic-size matrix types. For fixed-size types,
+ * it is redundant to pass \a rows and \a cols as arguments, so Ones() should be used
+ * instead.
+ *
+ * Example: \include MatrixBase_ones_int_int.cpp
+ * Output: \verbinclude MatrixBase_ones_int_int.out
+ *
+ * \sa Ones(), Ones(Index), isOnes(), class Ones
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Ones(
+    Index rows, Index cols) {
+  return Constant(rows, cols, Scalar(1));
+}
+
+/** \returns an expression of a vector where all coefficients equal one.
+ *
+ * The parameter \a newSize is the size of the returned vector.
+ * Must be compatible with this MatrixBase type.
+ *
+ * \only_for_vectors
+ *
+ * This variant is meant to be used for dynamic-size vector types. For fixed-size types,
+ * it is redundant to pass \a size as argument, so Ones() should be used
+ * instead.
+ *
+ * Example: \include MatrixBase_ones_int.cpp
+ * Output: \verbinclude MatrixBase_ones_int.out
+ *
+ * \sa Ones(), Ones(Index,Index), isOnes(), class Ones
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Ones(
+    Index newSize) {
+  return Constant(newSize, Scalar(1));
+}
+
+/** \returns an expression of a fixed-size matrix or vector where all coefficients equal one.
+ *
+ * This variant is only for fixed-size MatrixBase types. For dynamic-size types, you
+ * need to use the variants taking size arguments.
+ *
+ * Example: \include MatrixBase_ones.cpp
+ * Output: \verbinclude MatrixBase_ones.out
+ *
+ * \sa Ones(Index), Ones(Index,Index), isOnes(), class Ones
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstantReturnType DenseBase<Derived>::Ones() {
+  return Constant(Scalar(1));
+}
+
+/** \returns true if *this is approximately equal to the matrix where all coefficients
+ *          are equal to 1, within the precision given by \a prec.
+ *
+ * Example: \include MatrixBase_isOnes.cpp
+ * Output: \verbinclude MatrixBase_isOnes.out
+ *
+ * \sa class CwiseNullaryOp, Ones()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isOnes(const RealScalar& prec) const {
+  return isApproxToConstant(Scalar(1), prec);
+}
+
+/** Sets all coefficients in this expression to one.
+ *
+ * Example: \include MatrixBase_setOnes.cpp
+ * Output: \verbinclude MatrixBase_setOnes.out
+ *
+ * \sa class CwiseNullaryOp, Ones()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::setOnes() {
+  return setConstant(Scalar(1));
+}
+
+/** Resizes to the given \a newSize, and sets all coefficients in this expression to one.
+ *
+ * \only_for_vectors
+ *
+ * Example: \include Matrix_setOnes_int.cpp
+ * Output: \verbinclude Matrix_setOnes_int.out
+ *
+ * \sa MatrixBase::setOnes(), setOnes(Index,Index), class CwiseNullaryOp, MatrixBase::Ones()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setOnes(Index newSize) {
+  resize(newSize);
+  return setConstant(Scalar(1));
+}
+
+/** Resizes to the given size, and sets all coefficients in this expression to one.
+ *
+ * \param rows the new number of rows
+ * \param cols the new number of columns
+ *
+ * Example: \include Matrix_setOnes_int_int.cpp
+ * Output: \verbinclude Matrix_setOnes_int_int.out
+ *
+ * \sa MatrixBase::setOnes(), setOnes(Index), class CwiseNullaryOp, MatrixBase::Ones()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setOnes(Index rows, Index cols) {
+  resize(rows, cols);
+  return setConstant(Scalar(1));
+}
+
+/** Resizes to the given size, changing only the number of rows, and sets all
+ * coefficients in this expression to one. For the parameter of type NoChange_t,
+ * just pass the special value \c NoChange.
+ *
+ * \sa MatrixBase::setOnes(), setOnes(Index), setOnes(Index, Index), setOnes(NoChange_t, Index), class CwiseNullaryOp,
+ * MatrixBase::Ones()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setOnes(Index rows, NoChange_t) {
+  return setOnes(rows, cols());
+}
+
+/** Resizes to the given size, changing only the number of columns, and sets all
+ * coefficients in this expression to one. For the parameter of type NoChange_t,
+ * just pass the special value \c NoChange.
+ *
+ * \sa MatrixBase::setOnes(), setOnes(Index), setOnes(Index, Index), setOnes(Index, NoChange_t) class CwiseNullaryOp,
+ * MatrixBase::Ones()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setOnes(NoChange_t, Index cols) {
+  return setOnes(rows(), cols);
+}
+
+// Identity:
+
+/** \returns an expression of the identity matrix (not necessarily square).
+ *
+ * The parameters \a rows and \a cols are the number of rows and of columns of
+ * the returned matrix. Must be compatible with this MatrixBase type.
+ *
+ * This variant is meant to be used for dynamic-size matrix types. For fixed-size types,
+ * it is redundant to pass \a rows and \a cols as arguments, so Identity() should be used
+ * instead.
+ *
+ * Example: \include MatrixBase_identity_int_int.cpp
+ * Output: \verbinclude MatrixBase_identity_int_int.out
+ *
+ * \sa Identity(), setIdentity(), isIdentity()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::IdentityReturnType
+MatrixBase<Derived>::Identity(Index rows, Index cols) {
+  return DenseBase<Derived>::NullaryExpr(rows, cols, internal::scalar_identity_op<Scalar>());
+}
+
+/** \returns an expression of the identity matrix (not necessarily square).
+ *
+ * This variant is only for fixed-size MatrixBase types. For dynamic-size types, you
+ * need to use the variant taking size arguments.
+ *
+ * Example: \include MatrixBase_identity.cpp
+ * Output: \verbinclude MatrixBase_identity.out
+ *
+ * \sa Identity(Index,Index), setIdentity(), isIdentity()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::IdentityReturnType
+MatrixBase<Derived>::Identity() {
+  EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived)
+  return MatrixBase<Derived>::NullaryExpr(RowsAtCompileTime, ColsAtCompileTime, internal::scalar_identity_op<Scalar>());
+}
+
+/** \returns true if *this is approximately equal to the identity matrix
+ *          (not necessarily square),
+ *          within the precision given by \a prec.
+ *
+ * Example: \include MatrixBase_isIdentity.cpp
+ * Output: \verbinclude MatrixBase_isIdentity.out
+ *
+ * \sa class CwiseNullaryOp, Identity(), Identity(Index,Index), setIdentity()
+ */
+template <typename Derived>
+bool MatrixBase<Derived>::isIdentity(const RealScalar& prec) const {
+  typename internal::nested_eval<Derived, 1>::type self(derived());
+  for (Index j = 0; j < cols(); ++j) {
+    for (Index i = 0; i < rows(); ++i) {
+      if (i == j) {
+        if (!internal::isApprox(self.coeff(i, j), static_cast<Scalar>(1), prec)) return false;
+      } else {
+        if (!internal::isMuchSmallerThan(self.coeff(i, j), static_cast<RealScalar>(1), prec)) return false;
+      }
+    }
+  }
+  return true;
+}
+
+namespace internal {
+
+template <typename Derived, bool Big = (Derived::SizeAtCompileTime >= 16)>
+struct setIdentity_impl {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Derived& run(Derived& m) {
+    return m = Derived::Identity(m.rows(), m.cols());
+  }
+};
+
+template <typename Derived>
+struct setIdentity_impl<Derived, true> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Derived& run(Derived& m) {
+    m.setZero();
+    const Index size = numext::mini(m.rows(), m.cols());
+    for (Index i = 0; i < size; ++i) m.coeffRef(i, i) = typename Derived::Scalar(1);
+    return m;
+  }
+};
+
+}  // end namespace internal
+
+/** Writes the identity expression (not necessarily square) into *this.
+ *
+ * Example: \include MatrixBase_setIdentity.cpp
+ * Output: \verbinclude MatrixBase_setIdentity.out
+ *
+ * \sa class CwiseNullaryOp, Identity(), Identity(Index,Index), isIdentity()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::setIdentity() {
+  return internal::setIdentity_impl<Derived>::run(derived());
+}
+
+/** \brief Resizes to the given size, and writes the identity expression (not necessarily square) into *this.
+ *
+ * \param rows the new number of rows
+ * \param cols the new number of columns
+ *
+ * Example: \include Matrix_setIdentity_int_int.cpp
+ * Output: \verbinclude Matrix_setIdentity_int_int.out
+ *
+ * \sa MatrixBase::setIdentity(), class CwiseNullaryOp, MatrixBase::Identity()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::setIdentity(Index rows, Index cols) {
+  derived().resize(rows, cols);
+  return setIdentity();
+}
+
+/** \returns an expression of the i-th unit (basis) vector.
+ *
+ * \only_for_vectors
+ *
+ * \sa MatrixBase::Unit(Index), MatrixBase::UnitX(), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::Unit(
+    Index newSize, Index i) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  return BasisReturnType(SquareMatrixType::Identity(newSize, newSize), i);
+}
+
+/** \returns an expression of the i-th unit (basis) vector.
+ *
+ * \only_for_vectors
+ *
+ * This variant is for fixed-size vector only.
+ *
+ * \sa MatrixBase::Unit(Index,Index), MatrixBase::UnitX(), MatrixBase::UnitY(), MatrixBase::UnitZ(), MatrixBase::UnitW()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::Unit(
+    Index i) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  return BasisReturnType(SquareMatrixType::Identity(), i);
+}
+
+/** \returns an expression of the X axis unit vector (1{,0}^*)
+ *
+ * \only_for_vectors
+ *
+ * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(),
+ * MatrixBase::UnitW()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitX() {
+  return Derived::Unit(0);
+}
+
+/** \returns an expression of the Y axis unit vector (0,1{,0}^*)
+ *
+ * \only_for_vectors
+ *
+ * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(),
+ * MatrixBase::UnitW()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitY() {
+  return Derived::Unit(1);
+}
+
+/** \returns an expression of the Z axis unit vector (0,0,1{,0}^*)
+ *
+ * \only_for_vectors
+ *
+ * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(),
+ * MatrixBase::UnitW()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitZ() {
+  return Derived::Unit(2);
+}
+
+/** \returns an expression of the W axis unit vector (0,0,0,1)
+ *
+ * \only_for_vectors
+ *
+ * \sa MatrixBase::Unit(Index,Index), MatrixBase::Unit(Index), MatrixBase::UnitY(), MatrixBase::UnitZ(),
+ * MatrixBase::UnitW()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::BasisReturnType MatrixBase<Derived>::UnitW() {
+  return Derived::Unit(3);
+}
+
+/** \brief Set the coefficients of \c *this to the i-th unit (basis) vector
+ *
+ * \param i index of the unique coefficient to be set to 1
+ *
+ * \only_for_vectors
+ *
+ * \sa MatrixBase::setIdentity(), class CwiseNullaryOp, MatrixBase::Unit(Index,Index)
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::setUnit(Index i) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived);
+  eigen_assert(i < size());
+  derived().setZero();
+  derived().coeffRef(i) = Scalar(1);
+  return derived();
+}
+
+/** \brief Resizes to the given \a newSize, and writes the i-th unit (basis) vector into *this.
+ *
+ * \param newSize the new size of the vector
+ * \param i index of the unique coefficient to be set to 1
+ *
+ * \only_for_vectors
+ *
+ * \sa MatrixBase::setIdentity(), class CwiseNullaryOp, MatrixBase::Unit(Index,Index)
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::setUnit(Index newSize, Index i) {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived);
+  eigen_assert(i < newSize);
+  derived().resize(newSize);
+  return setUnit(i);
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_CWISE_NULLARY_OP_H

include/Eigen/src/Core/CwiseTernaryOp.h

@@ -0,0 +1,171 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2016 Eugene Brevdo <ebrevdo@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_CWISE_TERNARY_OP_H
+#define EIGEN_CWISE_TERNARY_OP_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename TernaryOp, typename Arg1, typename Arg2, typename Arg3>
+struct traits<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>> {
+  // we must not inherit from traits<Arg1> since it has
+  // the potential to cause problems with MSVC
+  typedef remove_all_t<Arg1> Ancestor;
+  typedef typename traits<Ancestor>::XprKind XprKind;
+  enum {
+    RowsAtCompileTime = traits<Ancestor>::RowsAtCompileTime,
+    ColsAtCompileTime = traits<Ancestor>::ColsAtCompileTime,
+    MaxRowsAtCompileTime = traits<Ancestor>::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = traits<Ancestor>::MaxColsAtCompileTime
+  };
+
+  // even though we require Arg1, Arg2, and Arg3 to have the same scalar type
+  // (see CwiseTernaryOp constructor),
+  // we still want to handle the case when the result type is different.
+  typedef typename result_of<TernaryOp(const typename Arg1::Scalar&, const typename Arg2::Scalar&,
+                                       const typename Arg3::Scalar&)>::type Scalar;
+
+  typedef typename internal::traits<Arg1>::StorageKind StorageKind;
+  typedef typename internal::traits<Arg1>::StorageIndex StorageIndex;
+
+  typedef typename Arg1::Nested Arg1Nested;
+  typedef typename Arg2::Nested Arg2Nested;
+  typedef typename Arg3::Nested Arg3Nested;
+  typedef std::remove_reference_t<Arg1Nested> Arg1Nested_;
+  typedef std::remove_reference_t<Arg2Nested> Arg2Nested_;
+  typedef std::remove_reference_t<Arg3Nested> Arg3Nested_;
+  enum { Flags = Arg1Nested_::Flags & RowMajorBit };
+};
+}  // end namespace internal
+
+template <typename TernaryOp, typename Arg1, typename Arg2, typename Arg3, typename StorageKind>
+class CwiseTernaryOpImpl;
+
+/** \class CwiseTernaryOp
+ * \ingroup Core_Module
+ *
+ * \brief Generic expression where a coefficient-wise ternary operator is
+ * applied to two expressions
+ *
+ * \tparam TernaryOp template functor implementing the operator
+ * \tparam Arg1Type the type of the first argument
+ * \tparam Arg2Type the type of the second argument
+ * \tparam Arg3Type the type of the third argument
+ *
+ * This class represents an expression where a coefficient-wise ternary
+ * operator is applied to three expressions.
+ * It is the return type of ternary operators, by which we mean only those
+ * ternary operators where
+ * all three arguments are Eigen expressions.
+ * For example, the return type of betainc(matrix1, matrix2, matrix3) is a
+ * CwiseTernaryOp.
+ *
+ * Most of the time, this is the only way that it is used, so you typically
+ * don't have to name
+ * CwiseTernaryOp types explicitly.
+ *
+ * \sa MatrixBase::ternaryExpr(const MatrixBase<Argument2> &, const
+ * MatrixBase<Argument3> &, const CustomTernaryOp &) const, class CwiseBinaryOp,
+ * class CwiseUnaryOp, class CwiseNullaryOp
+ */
+template <typename TernaryOp, typename Arg1Type, typename Arg2Type, typename Arg3Type>
+class CwiseTernaryOp : public CwiseTernaryOpImpl<TernaryOp, Arg1Type, Arg2Type, Arg3Type,
+                                                 typename internal::traits<Arg1Type>::StorageKind>,
+                       internal::no_assignment_operator {
+ public:
+  typedef internal::remove_all_t<Arg1Type> Arg1;
+  typedef internal::remove_all_t<Arg2Type> Arg2;
+  typedef internal::remove_all_t<Arg3Type> Arg3;
+
+  // require the sizes to match
+  EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Arg1, Arg2)
+  EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Arg1, Arg3)
+
+  // The index types should match
+  EIGEN_STATIC_ASSERT((internal::is_same<typename internal::traits<Arg1Type>::StorageKind,
+                                         typename internal::traits<Arg2Type>::StorageKind>::value),
+                      STORAGE_KIND_MUST_MATCH)
+  EIGEN_STATIC_ASSERT((internal::is_same<typename internal::traits<Arg1Type>::StorageKind,
+                                         typename internal::traits<Arg3Type>::StorageKind>::value),
+                      STORAGE_KIND_MUST_MATCH)
+
+  typedef typename CwiseTernaryOpImpl<TernaryOp, Arg1Type, Arg2Type, Arg3Type,
+                                      typename internal::traits<Arg1Type>::StorageKind>::Base Base;
+  EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseTernaryOp)
+
+  typedef typename internal::ref_selector<Arg1Type>::type Arg1Nested;
+  typedef typename internal::ref_selector<Arg2Type>::type Arg2Nested;
+  typedef typename internal::ref_selector<Arg3Type>::type Arg3Nested;
+  typedef std::remove_reference_t<Arg1Nested> Arg1Nested_;
+  typedef std::remove_reference_t<Arg2Nested> Arg2Nested_;
+  typedef std::remove_reference_t<Arg3Nested> Arg3Nested_;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CwiseTernaryOp(const Arg1& a1, const Arg2& a2, const Arg3& a3,
+                                                       const TernaryOp& func = TernaryOp())
+      : m_arg1(a1), m_arg2(a2), m_arg3(a3), m_functor(func) {
+    eigen_assert(a1.rows() == a2.rows() && a1.cols() == a2.cols() && a1.rows() == a3.rows() && a1.cols() == a3.cols());
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rows() const {
+    // return the fixed size type if available to enable compile time
+    // optimizations
+    if (internal::traits<internal::remove_all_t<Arg1Nested>>::RowsAtCompileTime == Dynamic &&
+        internal::traits<internal::remove_all_t<Arg2Nested>>::RowsAtCompileTime == Dynamic)
+      return m_arg3.rows();
+    else if (internal::traits<internal::remove_all_t<Arg1Nested>>::RowsAtCompileTime == Dynamic &&
+             internal::traits<internal::remove_all_t<Arg3Nested>>::RowsAtCompileTime == Dynamic)
+      return m_arg2.rows();
+    else
+      return m_arg1.rows();
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index cols() const {
+    // return the fixed size type if available to enable compile time
+    // optimizations
+    if (internal::traits<internal::remove_all_t<Arg1Nested>>::ColsAtCompileTime == Dynamic &&
+        internal::traits<internal::remove_all_t<Arg2Nested>>::ColsAtCompileTime == Dynamic)
+      return m_arg3.cols();
+    else if (internal::traits<internal::remove_all_t<Arg1Nested>>::ColsAtCompileTime == Dynamic &&
+             internal::traits<internal::remove_all_t<Arg3Nested>>::ColsAtCompileTime == Dynamic)
+      return m_arg2.cols();
+    else
+      return m_arg1.cols();
+  }
+
+  /** \returns the first argument nested expression */
+  EIGEN_DEVICE_FUNC const Arg1Nested_& arg1() const { return m_arg1; }
+  /** \returns the first argument nested expression */
+  EIGEN_DEVICE_FUNC const Arg2Nested_& arg2() const { return m_arg2; }
+  /** \returns the third argument nested expression */
+  EIGEN_DEVICE_FUNC const Arg3Nested_& arg3() const { return m_arg3; }
+  /** \returns the functor representing the ternary operation */
+  EIGEN_DEVICE_FUNC const TernaryOp& functor() const { return m_functor; }
+
+ protected:
+  Arg1Nested m_arg1;
+  Arg2Nested m_arg2;
+  Arg3Nested m_arg3;
+  const TernaryOp m_functor;
+};
+
+// Generic API dispatcher
+template <typename TernaryOp, typename Arg1, typename Arg2, typename Arg3, typename StorageKind>
+class CwiseTernaryOpImpl : public internal::generic_xpr_base<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>>::type {
+ public:
+  typedef typename internal::generic_xpr_base<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>>::type Base;
+};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_CWISE_TERNARY_OP_H

include/Eigen/src/Core/CwiseUnaryOp.h

@@ -0,0 +1,91 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_CWISE_UNARY_OP_H
+#define EIGEN_CWISE_UNARY_OP_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename UnaryOp, typename XprType>
+struct traits<CwiseUnaryOp<UnaryOp, XprType> > : traits<XprType> {
+  typedef typename result_of<UnaryOp(const typename XprType::Scalar&)>::type Scalar;
+  typedef typename XprType::Nested XprTypeNested;
+  typedef std::remove_reference_t<XprTypeNested> XprTypeNested_;
+  enum { Flags = XprTypeNested_::Flags & RowMajorBit };
+};
+}  // namespace internal
+
+template <typename UnaryOp, typename XprType, typename StorageKind>
+class CwiseUnaryOpImpl;
+
+/** \class CwiseUnaryOp
+ * \ingroup Core_Module
+ *
+ * \brief Generic expression where a coefficient-wise unary operator is applied to an expression
+ *
+ * \tparam UnaryOp template functor implementing the operator
+ * \tparam XprType the type of the expression to which we are applying the unary operator
+ *
+ * This class represents an expression where a unary operator is applied to an expression.
+ * It is the return type of all operations taking exactly 1 input expression, regardless of the
+ * presence of other inputs such as scalars. For example, the operator* in the expression 3*matrix
+ * is considered unary, because only the right-hand side is an expression, and its
+ * return type is a specialization of CwiseUnaryOp.
+ *
+ * Most of the time, this is the only way that it is used, so you typically don't have to name
+ * CwiseUnaryOp types explicitly.
+ *
+ * \sa MatrixBase::unaryExpr(const CustomUnaryOp &) const, class CwiseBinaryOp, class CwiseNullaryOp
+ */
+template <typename UnaryOp, typename XprType>
+class CwiseUnaryOp : public CwiseUnaryOpImpl<UnaryOp, XprType, typename internal::traits<XprType>::StorageKind>,
+                     internal::no_assignment_operator {
+ public:
+  typedef typename CwiseUnaryOpImpl<UnaryOp, XprType, typename internal::traits<XprType>::StorageKind>::Base Base;
+  EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseUnaryOp)
+  typedef typename internal::ref_selector<XprType>::type XprTypeNested;
+  typedef internal::remove_all_t<XprType> NestedExpression;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit CwiseUnaryOp(const XprType& xpr, const UnaryOp& func = UnaryOp())
+      : m_xpr(xpr), m_functor(func) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_xpr.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_xpr.cols(); }
+
+  /** \returns the functor representing the unary operation */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const UnaryOp& functor() const { return m_functor; }
+
+  /** \returns the nested expression */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const internal::remove_all_t<XprTypeNested>& nestedExpression() const {
+    return m_xpr;
+  }
+
+  /** \returns the nested expression */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::remove_all_t<XprTypeNested>& nestedExpression() { return m_xpr; }
+
+ protected:
+  XprTypeNested m_xpr;
+  const UnaryOp m_functor;
+};
+
+// Generic API dispatcher
+template <typename UnaryOp, typename XprType, typename StorageKind>
+class CwiseUnaryOpImpl : public internal::generic_xpr_base<CwiseUnaryOp<UnaryOp, XprType> >::type {
+ public:
+  typedef typename internal::generic_xpr_base<CwiseUnaryOp<UnaryOp, XprType> >::type Base;
+};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_CWISE_UNARY_OP_H

include/Eigen/src/Core/CwiseUnaryView.h

@@ -0,0 +1,167 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_CWISE_UNARY_VIEW_H
+#define EIGEN_CWISE_UNARY_VIEW_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename ViewOp, typename MatrixType, typename StrideType>
+struct traits<CwiseUnaryView<ViewOp, MatrixType, StrideType> > : traits<MatrixType> {
+  typedef typename result_of<ViewOp(typename traits<MatrixType>::Scalar&)>::type1 ScalarRef;
+  static_assert(std::is_reference<ScalarRef>::value, "Views must return a reference type.");
+  typedef remove_all_t<ScalarRef> Scalar;
+  typedef typename MatrixType::Nested MatrixTypeNested;
+  typedef remove_all_t<MatrixTypeNested> MatrixTypeNested_;
+  enum {
+    FlagsLvalueBit = is_lvalue<MatrixType>::value ? LvalueBit : 0,
+    Flags =
+        traits<MatrixTypeNested_>::Flags &
+        (RowMajorBit | FlagsLvalueBit | DirectAccessBit),  // FIXME DirectAccessBit should not be handled by expressions
+    MatrixTypeInnerStride = inner_stride_at_compile_time<MatrixType>::ret,
+    // need to cast the sizeof's from size_t to int explicitly, otherwise:
+    // "error: no integral type can represent all of the enumerator values
+    InnerStrideAtCompileTime =
+        StrideType::InnerStrideAtCompileTime == 0
+            ? (MatrixTypeInnerStride == Dynamic
+                   ? int(Dynamic)
+                   : int(MatrixTypeInnerStride) * int(sizeof(typename traits<MatrixType>::Scalar) / sizeof(Scalar)))
+            : int(StrideType::InnerStrideAtCompileTime),
+
+    OuterStrideAtCompileTime = StrideType::OuterStrideAtCompileTime == 0
+                                   ? (outer_stride_at_compile_time<MatrixType>::ret == Dynamic
+                                          ? int(Dynamic)
+                                          : outer_stride_at_compile_time<MatrixType>::ret *
+                                                int(sizeof(typename traits<MatrixType>::Scalar) / sizeof(Scalar)))
+                                   : int(StrideType::OuterStrideAtCompileTime)
+  };
+};
+
+// Generic API dispatcher
+template <typename ViewOp, typename XprType, typename StrideType, typename StorageKind,
+          bool Mutable = !std::is_const<XprType>::value>
+class CwiseUnaryViewImpl : public generic_xpr_base<CwiseUnaryView<ViewOp, XprType, StrideType> >::type {
+ public:
+  typedef typename generic_xpr_base<CwiseUnaryView<ViewOp, XprType, StrideType> >::type Base;
+};
+
+template <typename ViewOp, typename MatrixType, typename StrideType>
+class CwiseUnaryViewImpl<ViewOp, MatrixType, StrideType, Dense, false>
+    : public dense_xpr_base<CwiseUnaryView<ViewOp, MatrixType, StrideType> >::type {
+ public:
+  typedef CwiseUnaryView<ViewOp, MatrixType, StrideType> Derived;
+  typedef typename dense_xpr_base<CwiseUnaryView<ViewOp, MatrixType, StrideType> >::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Derived)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(CwiseUnaryViewImpl)
+
+  EIGEN_DEVICE_FUNC inline const Scalar* data() const { return &(this->coeffRef(0)); }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const {
+    return StrideType::InnerStrideAtCompileTime != 0 ? int(StrideType::InnerStrideAtCompileTime)
+                                                     : derived().nestedExpression().innerStride() *
+                                                           sizeof(typename traits<MatrixType>::Scalar) / sizeof(Scalar);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const {
+    return StrideType::OuterStrideAtCompileTime != 0 ? int(StrideType::OuterStrideAtCompileTime)
+                                                     : derived().nestedExpression().outerStride() *
+                                                           sizeof(typename traits<MatrixType>::Scalar) / sizeof(Scalar);
+  }
+
+ protected:
+  EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(CwiseUnaryViewImpl)
+
+  // Allow const access to coeffRef for the case of direct access being enabled.
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const {
+    return internal::evaluator<Derived>(derived()).coeffRef(index);
+  }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index row, Index col) const {
+    return internal::evaluator<Derived>(derived()).coeffRef(row, col);
+  }
+};
+
+template <typename ViewOp, typename MatrixType, typename StrideType>
+class CwiseUnaryViewImpl<ViewOp, MatrixType, StrideType, Dense, true>
+    : public CwiseUnaryViewImpl<ViewOp, MatrixType, StrideType, Dense, false> {
+ public:
+  typedef CwiseUnaryViewImpl<ViewOp, MatrixType, StrideType, Dense, false> Base;
+  typedef CwiseUnaryView<ViewOp, MatrixType, StrideType> Derived;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Derived)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(CwiseUnaryViewImpl)
+
+  using Base::data;
+  EIGEN_DEVICE_FUNC inline Scalar* data() { return &(this->coeffRef(0)); }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) {
+    return internal::evaluator<Derived>(derived()).coeffRef(row, col);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) {
+    return internal::evaluator<Derived>(derived()).coeffRef(index);
+  }
+
+ protected:
+  EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(CwiseUnaryViewImpl)
+};
+
+}  // namespace internal
+
+/** \class CwiseUnaryView
+ * \ingroup Core_Module
+ *
+ * \brief Generic lvalue expression of a coefficient-wise unary operator of a matrix or a vector
+ *
+ * \tparam ViewOp template functor implementing the view
+ * \tparam MatrixType the type of the matrix we are applying the unary operator
+ *
+ * This class represents a lvalue expression of a generic unary view operator of a matrix or a vector.
+ * It is the return type of real() and imag(), and most of the time this is the only way it is used.
+ *
+ * \sa MatrixBase::unaryViewExpr(const CustomUnaryOp &) const, class CwiseUnaryOp
+ */
+template <typename ViewOp, typename MatrixType, typename StrideType>
+class CwiseUnaryView : public internal::CwiseUnaryViewImpl<ViewOp, MatrixType, StrideType,
+                                                           typename internal::traits<MatrixType>::StorageKind> {
+ public:
+  typedef typename internal::CwiseUnaryViewImpl<ViewOp, MatrixType, StrideType,
+                                                typename internal::traits<MatrixType>::StorageKind>::Base Base;
+  EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseUnaryView)
+  typedef typename internal::ref_selector<MatrixType>::non_const_type MatrixTypeNested;
+  typedef internal::remove_all_t<MatrixType> NestedExpression;
+
+  explicit EIGEN_DEVICE_FUNC inline CwiseUnaryView(MatrixType& mat, const ViewOp& func = ViewOp())
+      : m_matrix(mat), m_functor(func) {}
+
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(CwiseUnaryView)
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_matrix.cols(); }
+
+  /** \returns the functor representing unary operation */
+  EIGEN_DEVICE_FUNC const ViewOp& functor() const { return m_functor; }
+
+  /** \returns the nested expression */
+  EIGEN_DEVICE_FUNC const internal::remove_all_t<MatrixTypeNested>& nestedExpression() const { return m_matrix; }
+
+  /** \returns the nested expression */
+  EIGEN_DEVICE_FUNC std::remove_reference_t<MatrixTypeNested>& nestedExpression() { return m_matrix; }
+
+ protected:
+  MatrixTypeNested m_matrix;
+  ViewOp m_functor;
+};
+
+}  // namespace Eigen
+
+#endif  // EIGEN_CWISE_UNARY_VIEW_H

include/Eigen/src/Core/DenseBase.h

@@ -0,0 +1,661 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2007-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_DENSEBASE_H
+#define EIGEN_DENSEBASE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+// The index type defined by EIGEN_DEFAULT_DENSE_INDEX_TYPE must be a signed type.
+EIGEN_STATIC_ASSERT(NumTraits<DenseIndex>::IsSigned, THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE)
+
+/** \class DenseBase
+ * \ingroup Core_Module
+ *
+ * \brief Base class for all dense matrices, vectors, and arrays
+ *
+ * This class is the base that is inherited by all dense objects (matrix, vector, arrays,
+ * and related expression types). The common Eigen API for dense objects is contained in this class.
+ *
+ * \tparam Derived is the derived type, e.g., a matrix type or an expression.
+ *
+ * This class can be extended with the help of the plugin mechanism described on the page
+ * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_DENSEBASE_PLUGIN.
+ *
+ * \sa \blank \ref TopicClassHierarchy
+ */
+template <typename Derived>
+class DenseBase
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+    : public DenseCoeffsBase<Derived, internal::accessors_level<Derived>::value>
+#else
+    : public DenseCoeffsBase<Derived, DirectWriteAccessors>
+#endif  // not EIGEN_PARSED_BY_DOXYGEN
+{
+ public:
+  /** Inner iterator type to iterate over the coefficients of a row or column.
+   * \sa class InnerIterator
+   */
+  typedef Eigen::InnerIterator<Derived> InnerIterator;
+
+  typedef typename internal::traits<Derived>::StorageKind StorageKind;
+
+  /**
+   * \brief The type used to store indices
+   * \details This typedef is relevant for types that store multiple indices such as
+   *          PermutationMatrix or Transpositions, otherwise it defaults to Eigen::Index
+   * \sa \blank \ref TopicPreprocessorDirectives, Eigen::Index, SparseMatrixBase.
+   */
+  typedef typename internal::traits<Derived>::StorageIndex StorageIndex;
+
+  /** The numeric type of the expression' coefficients, e.g. float, double, int or std::complex<float>, etc. */
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+
+  /** The numeric type of the expression' coefficients, e.g. float, double, int or std::complex<float>, etc.
+   *
+   * It is an alias for the Scalar type */
+  typedef Scalar value_type;
+
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+  typedef DenseCoeffsBase<Derived, internal::accessors_level<Derived>::value> Base;
+
+  using Base::coeff;
+  using Base::coeffByOuterInner;
+  using Base::colIndexByOuterInner;
+  using Base::cols;
+  using Base::const_cast_derived;
+  using Base::derived;
+  using Base::rowIndexByOuterInner;
+  using Base::rows;
+  using Base::size;
+  using Base::operator();
+  using Base::operator[];
+  using Base::colStride;
+  using Base::innerStride;
+  using Base::outerStride;
+  using Base::rowStride;
+  using Base::stride;
+  using Base::w;
+  using Base::x;
+  using Base::y;
+  using Base::z;
+  typedef typename Base::CoeffReturnType CoeffReturnType;
+
+  enum {
+
+    RowsAtCompileTime = internal::traits<Derived>::RowsAtCompileTime,
+    /**< The number of rows at compile-time. This is just a copy of the value provided
+     * by the \a Derived type. If a value is not known at compile-time,
+     * it is set to the \a Dynamic constant.
+     * \sa MatrixBase::rows(), MatrixBase::cols(), ColsAtCompileTime, SizeAtCompileTime */
+
+    ColsAtCompileTime = internal::traits<Derived>::ColsAtCompileTime,
+    /**< The number of columns at compile-time. This is just a copy of the value provided
+     * by the \a Derived type. If a value is not known at compile-time,
+     * it is set to the \a Dynamic constant.
+     * \sa MatrixBase::rows(), MatrixBase::cols(), RowsAtCompileTime, SizeAtCompileTime */
+
+    SizeAtCompileTime = (internal::size_of_xpr_at_compile_time<Derived>::ret),
+    /**< This is equal to the number of coefficients, i.e. the number of
+     * rows times the number of columns, or to \a Dynamic if this is not
+     * known at compile-time. \sa RowsAtCompileTime, ColsAtCompileTime */
+
+    MaxRowsAtCompileTime = internal::traits<Derived>::MaxRowsAtCompileTime,
+    /**< This value is equal to the maximum possible number of rows that this expression
+     * might have. If this expression might have an arbitrarily high number of rows,
+     * this value is set to \a Dynamic.
+     *
+     * This value is useful to know when evaluating an expression, in order to determine
+     * whether it is possible to avoid doing a dynamic memory allocation.
+     *
+     * \sa RowsAtCompileTime, MaxColsAtCompileTime, MaxSizeAtCompileTime
+     */
+
+    MaxColsAtCompileTime = internal::traits<Derived>::MaxColsAtCompileTime,
+    /**< This value is equal to the maximum possible number of columns that this expression
+     * might have. If this expression might have an arbitrarily high number of columns,
+     * this value is set to \a Dynamic.
+     *
+     * This value is useful to know when evaluating an expression, in order to determine
+     * whether it is possible to avoid doing a dynamic memory allocation.
+     *
+     * \sa ColsAtCompileTime, MaxRowsAtCompileTime, MaxSizeAtCompileTime
+     */
+
+    MaxSizeAtCompileTime = internal::size_at_compile_time(internal::traits<Derived>::MaxRowsAtCompileTime,
+                                                          internal::traits<Derived>::MaxColsAtCompileTime),
+    /**< This value is equal to the maximum possible number of coefficients that this expression
+     * might have. If this expression might have an arbitrarily high number of coefficients,
+     * this value is set to \a Dynamic.
+     *
+     * This value is useful to know when evaluating an expression, in order to determine
+     * whether it is possible to avoid doing a dynamic memory allocation.
+     *
+     * \sa SizeAtCompileTime, MaxRowsAtCompileTime, MaxColsAtCompileTime
+     */
+
+    IsVectorAtCompileTime =
+        internal::traits<Derived>::RowsAtCompileTime == 1 || internal::traits<Derived>::ColsAtCompileTime == 1,
+    /**< This is set to true if either the number of rows or the number of
+     * columns is known at compile-time to be equal to 1. Indeed, in that case,
+     * we are dealing with a column-vector (if there is only one column) or with
+     * a row-vector (if there is only one row). */
+
+    NumDimensions = int(MaxSizeAtCompileTime) == 1 ? 0
+                    : bool(IsVectorAtCompileTime)  ? 1
+                                                   : 2,
+    /**< This value is equal to Tensor::NumDimensions, i.e. 0 for scalars, 1 for vectors,
+     * and 2 for matrices.
+     */
+
+    Flags = internal::traits<Derived>::Flags,
+    /**< This stores expression \ref flags flags which may or may not be inherited by new expressions
+     * constructed from this one. See the \ref flags "list of flags".
+     */
+
+    IsRowMajor = int(Flags) & RowMajorBit, /**< True if this expression has row-major storage order. */
+
+    InnerSizeAtCompileTime = int(IsVectorAtCompileTime) ? int(SizeAtCompileTime)
+                             : int(IsRowMajor)          ? int(ColsAtCompileTime)
+                                                        : int(RowsAtCompileTime),
+
+    InnerStrideAtCompileTime = internal::inner_stride_at_compile_time<Derived>::ret,
+    OuterStrideAtCompileTime = internal::outer_stride_at_compile_time<Derived>::ret
+  };
+
+  typedef typename internal::find_best_packet<Scalar, SizeAtCompileTime>::type PacketScalar;
+
+  enum { IsPlainObjectBase = 0 };
+
+  /** The plain matrix type corresponding to this expression.
+   * \sa PlainObject */
+  typedef Matrix<typename internal::traits<Derived>::Scalar, internal::traits<Derived>::RowsAtCompileTime,
+                 internal::traits<Derived>::ColsAtCompileTime,
+                 AutoAlign | (internal::traits<Derived>::Flags & RowMajorBit ? RowMajor : ColMajor),
+                 internal::traits<Derived>::MaxRowsAtCompileTime, internal::traits<Derived>::MaxColsAtCompileTime>
+      PlainMatrix;
+
+  /** The plain array type corresponding to this expression.
+   * \sa PlainObject */
+  typedef Array<typename internal::traits<Derived>::Scalar, internal::traits<Derived>::RowsAtCompileTime,
+                internal::traits<Derived>::ColsAtCompileTime,
+                AutoAlign | (internal::traits<Derived>::Flags & RowMajorBit ? RowMajor : ColMajor),
+                internal::traits<Derived>::MaxRowsAtCompileTime, internal::traits<Derived>::MaxColsAtCompileTime>
+      PlainArray;
+
+  /** \brief The plain matrix or array type corresponding to this expression.
+   *
+   * This is not necessarily exactly the return type of eval(). In the case of plain matrices,
+   * the return type of eval() is a const reference to a matrix, not a matrix! It is however guaranteed
+   * that the return type of eval() is either PlainObject or const PlainObject&.
+   */
+  typedef std::conditional_t<internal::is_same<typename internal::traits<Derived>::XprKind, MatrixXpr>::value,
+                             PlainMatrix, PlainArray>
+      PlainObject;
+
+  /** \returns the outer size.
+   *
+   * \note For a vector, this returns just 1. For a matrix (non-vector), this is the major dimension
+   * with respect to the \ref TopicStorageOrders "storage order", i.e., the number of columns for a
+   * column-major matrix, and the number of rows for a row-major matrix. */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index outerSize() const {
+    return IsVectorAtCompileTime ? 1 : int(IsRowMajor) ? this->rows() : this->cols();
+  }
+
+  /** \returns the inner size.
+   *
+   * \note For a vector, this is just the size. For a matrix (non-vector), this is the minor dimension
+   * with respect to the \ref TopicStorageOrders "storage order", i.e., the number of rows for a
+   * column-major matrix, and the number of columns for a row-major matrix. */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index innerSize() const {
+    return IsVectorAtCompileTime ? this->size() : int(IsRowMajor) ? this->cols() : this->rows();
+  }
+
+  /** Only plain matrices/arrays, not expressions, may be resized; therefore the only useful resize methods are
+   * Matrix::resize() and Array::resize(). The present method only asserts that the new size equals the old size, and
+   * does nothing else.
+   */
+  EIGEN_DEVICE_FUNC void resize(Index newSize) {
+    EIGEN_ONLY_USED_FOR_DEBUG(newSize);
+    eigen_assert(newSize == this->size() && "DenseBase::resize() does not actually allow to resize.");
+  }
+  /** Only plain matrices/arrays, not expressions, may be resized; therefore the only useful resize methods are
+   * Matrix::resize() and Array::resize(). The present method only asserts that the new size equals the old size, and
+   * does nothing else.
+   */
+  EIGEN_DEVICE_FUNC void resize(Index rows, Index cols) {
+    EIGEN_ONLY_USED_FOR_DEBUG(rows);
+    EIGEN_ONLY_USED_FOR_DEBUG(cols);
+    eigen_assert(rows == this->rows() && cols == this->cols() &&
+                 "DenseBase::resize() does not actually allow to resize.");
+  }
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** \internal Represents a matrix with all coefficients equal to one another*/
+  typedef CwiseNullaryOp<internal::scalar_constant_op<Scalar>, PlainObject> ConstantReturnType;
+  /** \internal Represents a matrix with all coefficients equal to zero*/
+  typedef CwiseNullaryOp<internal::scalar_zero_op<Scalar>, PlainObject> ZeroReturnType;
+  /** \internal \deprecated Represents a vector with linearly spaced coefficients that allows sequential access only. */
+  EIGEN_DEPRECATED typedef CwiseNullaryOp<internal::linspaced_op<Scalar>, PlainObject> SequentialLinSpacedReturnType;
+  /** \internal Represents a vector with linearly spaced coefficients that allows random access. */
+  typedef CwiseNullaryOp<internal::linspaced_op<Scalar>, PlainObject> RandomAccessLinSpacedReturnType;
+  /** \internal Represents a vector with equally spaced coefficients that allows random access. */
+  typedef CwiseNullaryOp<internal::equalspaced_op<Scalar>, PlainObject> RandomAccessEqualSpacedReturnType;
+  /** \internal the return type of MatrixBase::eigenvalues() */
+  typedef Matrix<typename NumTraits<typename internal::traits<Derived>::Scalar>::Real,
+                 internal::traits<Derived>::ColsAtCompileTime, 1>
+      EigenvaluesReturnType;
+
+#endif  // not EIGEN_PARSED_BY_DOXYGEN
+
+  /** Copies \a other into *this. \returns a reference to *this. */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const DenseBase<OtherDerived>& other);
+
+  /** Special case of the template operator=, in order to prevent the compiler
+   * from generating a default operator= (issue hit with g++ 4.1)
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const DenseBase& other);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC Derived& operator=(const EigenBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC Derived& operator+=(const EigenBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC Derived& operator-=(const EigenBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC Derived& operator=(const ReturnByValue<OtherDerived>& func);
+
+  /** \internal
+   * Copies \a other into *this without evaluating other. \returns a reference to *this. */
+  template <typename OtherDerived>
+  /** \deprecated */
+  EIGEN_DEPRECATED EIGEN_DEVICE_FUNC Derived& lazyAssign(const DenseBase<OtherDerived>& other);
+
+  EIGEN_DEVICE_FUNC CommaInitializer<Derived> operator<<(const Scalar& s);
+
+  template <unsigned int Added, unsigned int Removed>
+  /** \deprecated it now returns \c *this */
+  EIGEN_DEPRECATED const Derived& flagged() const {
+    return derived();
+  }
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC CommaInitializer<Derived> operator<<(const DenseBase<OtherDerived>& other);
+
+  typedef Transpose<Derived> TransposeReturnType;
+  EIGEN_DEVICE_FUNC TransposeReturnType transpose();
+  typedef Transpose<const Derived> ConstTransposeReturnType;
+  EIGEN_DEVICE_FUNC const ConstTransposeReturnType transpose() const;
+  EIGEN_DEVICE_FUNC void transposeInPlace();
+
+  EIGEN_DEVICE_FUNC static const ConstantReturnType Constant(Index rows, Index cols, const Scalar& value);
+  EIGEN_DEVICE_FUNC static const ConstantReturnType Constant(Index size, const Scalar& value);
+  EIGEN_DEVICE_FUNC static const ConstantReturnType Constant(const Scalar& value);
+
+  EIGEN_DEPRECATED EIGEN_DEVICE_FUNC static const RandomAccessLinSpacedReturnType LinSpaced(Sequential_t, Index size,
+                                                                                            const Scalar& low,
+                                                                                            const Scalar& high);
+  EIGEN_DEPRECATED EIGEN_DEVICE_FUNC static const RandomAccessLinSpacedReturnType LinSpaced(Sequential_t,
+                                                                                            const Scalar& low,
+                                                                                            const Scalar& high);
+
+  EIGEN_DEVICE_FUNC static const RandomAccessLinSpacedReturnType LinSpaced(Index size, const Scalar& low,
+                                                                           const Scalar& high);
+  EIGEN_DEVICE_FUNC static const RandomAccessLinSpacedReturnType LinSpaced(const Scalar& low, const Scalar& high);
+
+  EIGEN_DEVICE_FUNC static const RandomAccessEqualSpacedReturnType EqualSpaced(Index size, const Scalar& low,
+                                                                               const Scalar& step);
+  EIGEN_DEVICE_FUNC static const RandomAccessEqualSpacedReturnType EqualSpaced(const Scalar& low, const Scalar& step);
+
+  template <typename CustomNullaryOp>
+  EIGEN_DEVICE_FUNC static const CwiseNullaryOp<CustomNullaryOp, PlainObject> NullaryExpr(Index rows, Index cols,
+                                                                                          const CustomNullaryOp& func);
+  template <typename CustomNullaryOp>
+  EIGEN_DEVICE_FUNC static const CwiseNullaryOp<CustomNullaryOp, PlainObject> NullaryExpr(Index size,
+                                                                                          const CustomNullaryOp& func);
+  template <typename CustomNullaryOp>
+  EIGEN_DEVICE_FUNC static const CwiseNullaryOp<CustomNullaryOp, PlainObject> NullaryExpr(const CustomNullaryOp& func);
+
+  EIGEN_DEVICE_FUNC static const ZeroReturnType Zero(Index rows, Index cols);
+  EIGEN_DEVICE_FUNC static const ZeroReturnType Zero(Index size);
+  EIGEN_DEVICE_FUNC static const ZeroReturnType Zero();
+  EIGEN_DEVICE_FUNC static const ConstantReturnType Ones(Index rows, Index cols);
+  EIGEN_DEVICE_FUNC static const ConstantReturnType Ones(Index size);
+  EIGEN_DEVICE_FUNC static const ConstantReturnType Ones();
+
+  EIGEN_DEVICE_FUNC void fill(const Scalar& value);
+  EIGEN_DEVICE_FUNC Derived& setConstant(const Scalar& value);
+  EIGEN_DEVICE_FUNC Derived& setLinSpaced(Index size, const Scalar& low, const Scalar& high);
+  EIGEN_DEVICE_FUNC Derived& setLinSpaced(const Scalar& low, const Scalar& high);
+  EIGEN_DEVICE_FUNC Derived& setEqualSpaced(Index size, const Scalar& low, const Scalar& step);
+  EIGEN_DEVICE_FUNC Derived& setEqualSpaced(const Scalar& low, const Scalar& step);
+  EIGEN_DEVICE_FUNC Derived& setZero();
+  EIGEN_DEVICE_FUNC Derived& setOnes();
+  EIGEN_DEVICE_FUNC Derived& setRandom();
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC bool isApprox(const DenseBase<OtherDerived>& other,
+                                  const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+  EIGEN_DEVICE_FUNC bool isMuchSmallerThan(const RealScalar& other,
+                                           const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC bool isMuchSmallerThan(const DenseBase<OtherDerived>& other,
+                                           const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+
+  EIGEN_DEVICE_FUNC bool isApproxToConstant(const Scalar& value,
+                                            const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+  EIGEN_DEVICE_FUNC bool isConstant(const Scalar& value,
+                                    const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+  EIGEN_DEVICE_FUNC bool isZero(const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+  EIGEN_DEVICE_FUNC bool isOnes(const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+
+  EIGEN_DEVICE_FUNC inline bool hasNaN() const;
+  EIGEN_DEVICE_FUNC inline bool allFinite() const;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator*=(const Scalar& other);
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator/=(const Scalar& other);
+
+  typedef internal::add_const_on_value_type_t<typename internal::eval<Derived>::type> EvalReturnType;
+  /** \returns the matrix or vector obtained by evaluating this expression.
+   *
+   * Notice that in the case of a plain matrix or vector (not an expression) this function just returns
+   * a const reference, in order to avoid a useless copy.
+   *
+   * \warning Be careful with eval() and the auto C++ keyword, as detailed in this \link TopicPitfalls_auto_keyword page
+   * \endlink.
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EvalReturnType eval() const {
+    // Even though MSVC does not honor strong inlining when the return type
+    // is a dynamic matrix, we desperately need strong inlining for fixed
+    // size types on MSVC.
+    return typename internal::eval<Derived>::type(derived());
+  }
+
+  /** swaps *this with the expression \a other.
+   *
+   */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void swap(const DenseBase<OtherDerived>& other) {
+    EIGEN_STATIC_ASSERT(!OtherDerived::IsPlainObjectBase, THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY);
+    eigen_assert(rows() == other.rows() && cols() == other.cols());
+    call_assignment(derived(), other.const_cast_derived(), internal::swap_assign_op<Scalar>());
+  }
+
+  /** swaps *this with the matrix or array \a other.
+   *
+   */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void swap(PlainObjectBase<OtherDerived>& other) {
+    eigen_assert(rows() == other.rows() && cols() == other.cols());
+    call_assignment(derived(), other.derived(), internal::swap_assign_op<Scalar>());
+  }
+
+  EIGEN_DEVICE_FUNC inline const NestByValue<Derived> nestByValue() const;
+  EIGEN_DEVICE_FUNC inline const ForceAlignedAccess<Derived> forceAlignedAccess() const;
+  EIGEN_DEVICE_FUNC inline ForceAlignedAccess<Derived> forceAlignedAccess();
+  template <bool Enable>
+  EIGEN_DEVICE_FUNC inline const std::conditional_t<Enable, ForceAlignedAccess<Derived>, Derived&>
+  forceAlignedAccessIf() const;
+  template <bool Enable>
+  EIGEN_DEVICE_FUNC inline std::conditional_t<Enable, ForceAlignedAccess<Derived>, Derived&> forceAlignedAccessIf();
+
+  EIGEN_DEVICE_FUNC Scalar sum() const;
+  EIGEN_DEVICE_FUNC Scalar mean() const;
+  EIGEN_DEVICE_FUNC Scalar trace() const;
+
+  EIGEN_DEVICE_FUNC Scalar prod() const;
+
+  template <int NaNPropagation>
+  EIGEN_DEVICE_FUNC typename internal::traits<Derived>::Scalar minCoeff() const;
+  template <int NaNPropagation>
+  EIGEN_DEVICE_FUNC typename internal::traits<Derived>::Scalar maxCoeff() const;
+
+  // By default, the fastest version with undefined NaN propagation semantics is
+  // used.
+  // TODO(rmlarsen): Replace with default template argument when we move to
+  // c++11 or beyond.
+  EIGEN_DEVICE_FUNC inline typename internal::traits<Derived>::Scalar minCoeff() const {
+    return minCoeff<PropagateFast>();
+  }
+  EIGEN_DEVICE_FUNC inline typename internal::traits<Derived>::Scalar maxCoeff() const {
+    return maxCoeff<PropagateFast>();
+  }
+
+  template <int NaNPropagation, typename IndexType>
+  EIGEN_DEVICE_FUNC typename internal::traits<Derived>::Scalar minCoeff(IndexType* row, IndexType* col) const;
+  template <int NaNPropagation, typename IndexType>
+  EIGEN_DEVICE_FUNC typename internal::traits<Derived>::Scalar maxCoeff(IndexType* row, IndexType* col) const;
+  template <int NaNPropagation, typename IndexType>
+  EIGEN_DEVICE_FUNC typename internal::traits<Derived>::Scalar minCoeff(IndexType* index) const;
+  template <int NaNPropagation, typename IndexType>
+  EIGEN_DEVICE_FUNC typename internal::traits<Derived>::Scalar maxCoeff(IndexType* index) const;
+
+  // TODO(rmlarsen): Replace these methods with a default template argument.
+  template <typename IndexType>
+  EIGEN_DEVICE_FUNC inline typename internal::traits<Derived>::Scalar minCoeff(IndexType* row, IndexType* col) const {
+    return minCoeff<PropagateFast>(row, col);
+  }
+  template <typename IndexType>
+  EIGEN_DEVICE_FUNC inline typename internal::traits<Derived>::Scalar maxCoeff(IndexType* row, IndexType* col) const {
+    return maxCoeff<PropagateFast>(row, col);
+  }
+  template <typename IndexType>
+  EIGEN_DEVICE_FUNC inline typename internal::traits<Derived>::Scalar minCoeff(IndexType* index) const {
+    return minCoeff<PropagateFast>(index);
+  }
+  template <typename IndexType>
+  EIGEN_DEVICE_FUNC inline typename internal::traits<Derived>::Scalar maxCoeff(IndexType* index) const {
+    return maxCoeff<PropagateFast>(index);
+  }
+
+  template <typename BinaryOp>
+  EIGEN_DEVICE_FUNC Scalar redux(const BinaryOp& func) const;
+
+  template <typename Visitor>
+  EIGEN_DEVICE_FUNC void visit(Visitor& func) const;
+
+  /** \returns a WithFormat proxy object allowing to print a matrix the with given
+   * format \a fmt.
+   *
+   * See class IOFormat for some examples.
+   *
+   * \sa class IOFormat, class WithFormat
+   */
+  inline const WithFormat<Derived> format(const IOFormat& fmt) const { return WithFormat<Derived>(derived(), fmt); }
+
+  /** \returns the unique coefficient of a 1x1 expression */
+  EIGEN_DEVICE_FUNC CoeffReturnType value() const {
+    EIGEN_STATIC_ASSERT_SIZE_1x1(Derived) eigen_assert(this->rows() == 1 && this->cols() == 1);
+    return derived().coeff(0, 0);
+  }
+
+  EIGEN_DEVICE_FUNC bool all() const;
+  EIGEN_DEVICE_FUNC bool any() const;
+  EIGEN_DEVICE_FUNC Index count() const;
+
+  typedef VectorwiseOp<Derived, Horizontal> RowwiseReturnType;
+  typedef const VectorwiseOp<const Derived, Horizontal> ConstRowwiseReturnType;
+  typedef VectorwiseOp<Derived, Vertical> ColwiseReturnType;
+  typedef const VectorwiseOp<const Derived, Vertical> ConstColwiseReturnType;
+
+  /** \returns a VectorwiseOp wrapper of *this for broadcasting and partial reductions
+   *
+   * Example: \include MatrixBase_rowwise.cpp
+   * Output: \verbinclude MatrixBase_rowwise.out
+   *
+   * \sa colwise(), class VectorwiseOp, \ref TutorialReductionsVisitorsBroadcasting
+   */
+  // Code moved here due to a CUDA compiler bug
+  EIGEN_DEVICE_FUNC inline ConstRowwiseReturnType rowwise() const { return ConstRowwiseReturnType(derived()); }
+  EIGEN_DEVICE_FUNC RowwiseReturnType rowwise();
+
+  /** \returns a VectorwiseOp wrapper of *this broadcasting and partial reductions
+   *
+   * Example: \include MatrixBase_colwise.cpp
+   * Output: \verbinclude MatrixBase_colwise.out
+   *
+   * \sa rowwise(), class VectorwiseOp, \ref TutorialReductionsVisitorsBroadcasting
+   */
+  EIGEN_DEVICE_FUNC inline ConstColwiseReturnType colwise() const { return ConstColwiseReturnType(derived()); }
+  EIGEN_DEVICE_FUNC ColwiseReturnType colwise();
+
+  typedef CwiseNullaryOp<internal::scalar_random_op<Scalar>, PlainObject> RandomReturnType;
+  static const RandomReturnType Random(Index rows, Index cols);
+  static const RandomReturnType Random(Index size);
+  static const RandomReturnType Random();
+
+  template <typename ThenDerived, typename ElseDerived>
+  inline EIGEN_DEVICE_FUNC
+      CwiseTernaryOp<internal::scalar_boolean_select_op<typename DenseBase<ThenDerived>::Scalar,
+                                                        typename DenseBase<ElseDerived>::Scalar, Scalar>,
+                     ThenDerived, ElseDerived, Derived>
+      select(const DenseBase<ThenDerived>& thenMatrix, const DenseBase<ElseDerived>& elseMatrix) const;
+
+  template <typename ThenDerived>
+  inline EIGEN_DEVICE_FUNC
+      CwiseTernaryOp<internal::scalar_boolean_select_op<typename DenseBase<ThenDerived>::Scalar,
+                                                        typename DenseBase<ThenDerived>::Scalar, Scalar>,
+                     ThenDerived, typename DenseBase<ThenDerived>::ConstantReturnType, Derived>
+      select(const DenseBase<ThenDerived>& thenMatrix, const typename DenseBase<ThenDerived>::Scalar& elseScalar) const;
+
+  template <typename ElseDerived>
+  inline EIGEN_DEVICE_FUNC
+      CwiseTernaryOp<internal::scalar_boolean_select_op<typename DenseBase<ElseDerived>::Scalar,
+                                                        typename DenseBase<ElseDerived>::Scalar, Scalar>,
+                     typename DenseBase<ElseDerived>::ConstantReturnType, ElseDerived, Derived>
+      select(const typename DenseBase<ElseDerived>::Scalar& thenScalar, const DenseBase<ElseDerived>& elseMatrix) const;
+
+  template <int p>
+  RealScalar lpNorm() const;
+
+  template <int RowFactor, int ColFactor>
+  EIGEN_DEVICE_FUNC const Replicate<Derived, RowFactor, ColFactor> replicate() const;
+  /**
+   * \return an expression of the replication of \c *this
+   *
+   * Example: \include MatrixBase_replicate_int_int.cpp
+   * Output: \verbinclude MatrixBase_replicate_int_int.out
+   *
+   * \sa VectorwiseOp::replicate(), DenseBase::replicate<int,int>(), class Replicate
+   */
+  // Code moved here due to a CUDA compiler bug
+  EIGEN_DEVICE_FUNC const Replicate<Derived, Dynamic, Dynamic> replicate(Index rowFactor, Index colFactor) const {
+    return Replicate<Derived, Dynamic, Dynamic>(derived(), rowFactor, colFactor);
+  }
+
+  typedef Reverse<Derived, BothDirections> ReverseReturnType;
+  typedef const Reverse<const Derived, BothDirections> ConstReverseReturnType;
+  EIGEN_DEVICE_FUNC ReverseReturnType reverse();
+  /** This is the const version of reverse(). */
+  // Code moved here due to a CUDA compiler bug
+  EIGEN_DEVICE_FUNC ConstReverseReturnType reverse() const { return ConstReverseReturnType(derived()); }
+  EIGEN_DEVICE_FUNC void reverseInPlace();
+
+#ifdef EIGEN_PARSED_BY_DOXYGEN
+  /** STL-like <a href="https://en.cppreference.com/w/cpp/named_req/RandomAccessIterator">RandomAccessIterator</a>
+   * iterator type as returned by the begin() and end() methods.
+   */
+  typedef random_access_iterator_type iterator;
+  /** This is the const version of iterator (aka read-only) */
+  typedef random_access_iterator_type const_iterator;
+#else
+  typedef std::conditional_t<(Flags & DirectAccessBit) == DirectAccessBit,
+                             internal::pointer_based_stl_iterator<Derived>,
+                             internal::generic_randaccess_stl_iterator<Derived> >
+      iterator_type;
+
+  typedef std::conditional_t<(Flags & DirectAccessBit) == DirectAccessBit,
+                             internal::pointer_based_stl_iterator<const Derived>,
+                             internal::generic_randaccess_stl_iterator<const Derived> >
+      const_iterator_type;
+
+  // Stl-style iterators are supported only for vectors.
+
+  typedef std::conditional_t<IsVectorAtCompileTime, iterator_type, void> iterator;
+
+  typedef std::conditional_t<IsVectorAtCompileTime, const_iterator_type, void> const_iterator;
+#endif
+
+  inline iterator begin();
+  inline const_iterator begin() const;
+  inline const_iterator cbegin() const;
+  inline iterator end();
+  inline const_iterator end() const;
+  inline const_iterator cend() const;
+
+#define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::DenseBase
+#define EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL
+#define EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF(COND)
+#define EIGEN_DOC_UNARY_ADDONS(X, Y)
+#include "../plugins/CommonCwiseUnaryOps.inc"
+#include "../plugins/BlockMethods.inc"
+#include "../plugins/IndexedViewMethods.inc"
+#include "../plugins/ReshapedMethods.inc"
+#ifdef EIGEN_DENSEBASE_PLUGIN
+#include EIGEN_DENSEBASE_PLUGIN
+#endif
+#undef EIGEN_CURRENT_STORAGE_BASE_CLASS
+#undef EIGEN_DOC_BLOCK_ADDONS_NOT_INNER_PANEL
+#undef EIGEN_DOC_BLOCK_ADDONS_INNER_PANEL_IF
+#undef EIGEN_DOC_UNARY_ADDONS
+
+  // disable the use of evalTo for dense objects with a nice compilation error
+  template <typename Dest>
+  EIGEN_DEVICE_FUNC inline void evalTo(Dest&) const {
+    EIGEN_STATIC_ASSERT((internal::is_same<Dest, void>::value),
+                        THE_EVAL_EVALTO_FUNCTION_SHOULD_NEVER_BE_CALLED_FOR_DENSE_OBJECTS);
+  }
+
+ protected:
+  EIGEN_DEFAULT_COPY_CONSTRUCTOR(DenseBase)
+  /** Default constructor. Do nothing. */
+#ifdef EIGEN_INTERNAL_DEBUGGING
+  EIGEN_DEVICE_FUNC constexpr DenseBase() {
+    /* Just checks for self-consistency of the flags.
+     * Only do it when debugging Eigen, as this borders on paranoia and could slow compilation down
+     */
+    EIGEN_STATIC_ASSERT(
+        (internal::check_implication(MaxRowsAtCompileTime == 1 && MaxColsAtCompileTime != 1, int(IsRowMajor)) &&
+         internal::check_implication(MaxColsAtCompileTime == 1 && MaxRowsAtCompileTime != 1, int(!IsRowMajor))),
+        INVALID_STORAGE_ORDER_FOR_THIS_VECTOR_EXPRESSION)
+  }
+#else
+  EIGEN_DEVICE_FUNC constexpr DenseBase() = default;
+#endif
+
+ private:
+  EIGEN_DEVICE_FUNC explicit DenseBase(int);
+  EIGEN_DEVICE_FUNC DenseBase(int, int);
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC explicit DenseBase(const DenseBase<OtherDerived>&);
+};
+
+/** Free-function swap.
+ */
+template <typename DerivedA, typename DerivedB>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
+    // Use forwarding references to capture all combinations of cv-qualified l+r-value cases.
+    std::enable_if_t<std::is_base_of<DenseBase<std::decay_t<DerivedA>>, std::decay_t<DerivedA>>::value &&
+                         std::is_base_of<DenseBase<std::decay_t<DerivedB>>, std::decay_t<DerivedB>>::value,
+                     void>
+    swap(DerivedA&& a, DerivedB&& b) {
+  a.swap(b);
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_DENSEBASE_H

include/Eigen/src/Core/DenseCoeffsBase.h

@@ -0,0 +1,569 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_DENSECOEFFSBASE_H
+#define EIGEN_DENSECOEFFSBASE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename T>
+struct add_const_on_value_type_if_arithmetic {
+  typedef std::conditional_t<is_arithmetic<T>::value, T, add_const_on_value_type_t<T>> type;
+};
+}  // namespace internal
+
+/** \brief Base class providing read-only coefficient access to matrices and arrays.
+ * \ingroup Core_Module
+ * \tparam Derived Type of the derived class
+ *
+ * \note #ReadOnlyAccessors Constant indicating read-only access
+ *
+ * This class defines the \c operator() \c const function and friends, which can be used to read specific
+ * entries of a matrix or array.
+ *
+ * \sa DenseCoeffsBase<Derived, WriteAccessors>, DenseCoeffsBase<Derived, DirectAccessors>,
+ *     \ref TopicClassHierarchy
+ */
+template <typename Derived>
+class DenseCoeffsBase<Derived, ReadOnlyAccessors> : public EigenBase<Derived> {
+ public:
+  typedef typename internal::traits<Derived>::StorageKind StorageKind;
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+  typedef typename internal::packet_traits<Scalar>::type PacketScalar;
+
+  // Explanation for this CoeffReturnType typedef.
+  // - This is the return type of the coeff() method.
+  // - The LvalueBit means exactly that we can offer a coeffRef() method, which means exactly that we can get references
+  // to coeffs, which means exactly that we can have coeff() return a const reference (as opposed to returning a value).
+  // - The is_arithmetic check is required since "const int", "const double", etc. will cause warnings on some systems
+  // while the declaration of "const T", where T is a non arithmetic type does not. Always returning "const Scalar&" is
+  // not possible, since the underlying expressions might not offer a valid address the reference could be referring to.
+  typedef std::conditional_t<bool(internal::traits<Derived>::Flags& LvalueBit), const Scalar&,
+                             std::conditional_t<internal::is_arithmetic<Scalar>::value, Scalar, const Scalar>>
+      CoeffReturnType;
+
+  typedef typename internal::add_const_on_value_type_if_arithmetic<typename internal::packet_traits<Scalar>::type>::type
+      PacketReturnType;
+
+  typedef EigenBase<Derived> Base;
+  using Base::cols;
+  using Base::derived;
+  using Base::rows;
+  using Base::size;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rowIndexByOuterInner(Index outer, Index inner) const {
+    return int(Derived::RowsAtCompileTime) == 1   ? 0
+           : int(Derived::ColsAtCompileTime) == 1 ? inner
+           : int(Derived::Flags) & RowMajorBit    ? outer
+                                                  : inner;
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index colIndexByOuterInner(Index outer, Index inner) const {
+    return int(Derived::ColsAtCompileTime) == 1   ? 0
+           : int(Derived::RowsAtCompileTime) == 1 ? inner
+           : int(Derived::Flags) & RowMajorBit    ? inner
+                                                  : outer;
+  }
+
+  /** Short version: don't use this function, use
+   * \link operator()(Index,Index) const \endlink instead.
+   *
+   * Long version: this function is similar to
+   * \link operator()(Index,Index) const \endlink, but without the assertion.
+   * Use this for limiting the performance cost of debugging code when doing
+   * repeated coefficient access. Only use this when it is guaranteed that the
+   * parameters \a row and \a col are in range.
+   *
+   * If EIGEN_INTERNAL_DEBUGGING is defined, an assertion will be made, making this
+   * function equivalent to \link operator()(Index,Index) const \endlink.
+   *
+   * \sa operator()(Index,Index) const, coeffRef(Index,Index), coeff(Index) const
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR CoeffReturnType coeff(Index row, Index col) const {
+    eigen_internal_assert(row >= 0 && row < rows() && col >= 0 && col < cols());
+    return internal::evaluator<Derived>(derived()).coeff(row, col);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR CoeffReturnType coeffByOuterInner(Index outer,
+                                                                                          Index inner) const {
+    return coeff(rowIndexByOuterInner(outer, inner), colIndexByOuterInner(outer, inner));
+  }
+
+  /** \returns the coefficient at given the given row and column.
+   *
+   * \sa operator()(Index,Index), operator[](Index)
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR CoeffReturnType operator()(Index row, Index col) const {
+    eigen_assert(row >= 0 && row < rows() && col >= 0 && col < cols());
+    return coeff(row, col);
+  }
+
+  /** Short version: don't use this function, use
+   * \link operator[](Index) const \endlink instead.
+   *
+   * Long version: this function is similar to
+   * \link operator[](Index) const \endlink, but without the assertion.
+   * Use this for limiting the performance cost of debugging code when doing
+   * repeated coefficient access. Only use this when it is guaranteed that the
+   * parameter \a index is in range.
+   *
+   * If EIGEN_INTERNAL_DEBUGGING is defined, an assertion will be made, making this
+   * function equivalent to \link operator[](Index) const \endlink.
+   *
+   * \sa operator[](Index) const, coeffRef(Index), coeff(Index,Index) const
+   */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR CoeffReturnType coeff(Index index) const {
+    EIGEN_STATIC_ASSERT(internal::evaluator<Derived>::Flags & LinearAccessBit,
+                        THIS_COEFFICIENT_ACCESSOR_TAKING_ONE_ACCESS_IS_ONLY_FOR_EXPRESSIONS_ALLOWING_LINEAR_ACCESS)
+    eigen_internal_assert(index >= 0 && index < size());
+    return internal::evaluator<Derived>(derived()).coeff(index);
+  }
+
+  /** \returns the coefficient at given index.
+   *
+   * This method is allowed only for vector expressions, and for matrix expressions having the LinearAccessBit.
+   *
+   * \sa operator[](Index), operator()(Index,Index) const, x() const, y() const,
+   * z() const, w() const
+   */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR CoeffReturnType operator[](Index index) const {
+    EIGEN_STATIC_ASSERT(Derived::IsVectorAtCompileTime,
+                        THE_BRACKET_OPERATOR_IS_ONLY_FOR_VECTORS__USE_THE_PARENTHESIS_OPERATOR_INSTEAD)
+    eigen_assert(index >= 0 && index < size());
+    return coeff(index);
+  }
+
+  /** \returns the coefficient at given index.
+   *
+   * This is synonymous to operator[](Index) const.
+   *
+   * This method is allowed only for vector expressions, and for matrix expressions having the LinearAccessBit.
+   *
+   * \sa operator[](Index), operator()(Index,Index) const, x() const, y() const,
+   * z() const, w() const
+   */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR CoeffReturnType operator()(Index index) const {
+    eigen_assert(index >= 0 && index < size());
+    return coeff(index);
+  }
+
+  /** equivalent to operator[](0).  */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR CoeffReturnType x() const { return (*this)[0]; }
+
+  /** equivalent to operator[](1).  */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR CoeffReturnType y() const {
+    EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime == -1 || Derived::SizeAtCompileTime >= 2, OUT_OF_RANGE_ACCESS);
+    return (*this)[1];
+  }
+
+  /** equivalent to operator[](2).  */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR CoeffReturnType z() const {
+    EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime == -1 || Derived::SizeAtCompileTime >= 3, OUT_OF_RANGE_ACCESS);
+    return (*this)[2];
+  }
+
+  /** equivalent to operator[](3).  */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR CoeffReturnType w() const {
+    EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime == -1 || Derived::SizeAtCompileTime >= 4, OUT_OF_RANGE_ACCESS);
+    return (*this)[3];
+  }
+
+  /** \internal
+   * \returns the packet of coefficients starting at the given row and column. It is your responsibility
+   * to ensure that a packet really starts there. This method is only available on expressions having the
+   * PacketAccessBit.
+   *
+   * The \a LoadMode parameter may have the value \a #Aligned or \a #Unaligned. Its effect is to select
+   * the appropriate vectorization instruction. Aligned access is faster, but is only possible for packets
+   * starting at an address which is a multiple of the packet size.
+   */
+
+  template <int LoadMode>
+  EIGEN_STRONG_INLINE PacketReturnType packet(Index row, Index col) const {
+    typedef typename internal::packet_traits<Scalar>::type DefaultPacketType;
+    eigen_internal_assert(row >= 0 && row < rows() && col >= 0 && col < cols());
+    return internal::evaluator<Derived>(derived()).template packet<LoadMode, DefaultPacketType>(row, col);
+  }
+
+  /** \internal */
+  template <int LoadMode>
+  EIGEN_STRONG_INLINE PacketReturnType packetByOuterInner(Index outer, Index inner) const {
+    return packet<LoadMode>(rowIndexByOuterInner(outer, inner), colIndexByOuterInner(outer, inner));
+  }
+
+  /** \internal
+   * \returns the packet of coefficients starting at the given index. It is your responsibility
+   * to ensure that a packet really starts there. This method is only available on expressions having the
+   * PacketAccessBit and the LinearAccessBit.
+   *
+   * The \a LoadMode parameter may have the value \a #Aligned or \a #Unaligned. Its effect is to select
+   * the appropriate vectorization instruction. Aligned access is faster, but is only possible for packets
+   * starting at an address which is a multiple of the packet size.
+   */
+
+  template <int LoadMode>
+  EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
+    EIGEN_STATIC_ASSERT(internal::evaluator<Derived>::Flags & LinearAccessBit,
+                        THIS_COEFFICIENT_ACCESSOR_TAKING_ONE_ACCESS_IS_ONLY_FOR_EXPRESSIONS_ALLOWING_LINEAR_ACCESS)
+    typedef typename internal::packet_traits<Scalar>::type DefaultPacketType;
+    eigen_internal_assert(index >= 0 && index < size());
+    return internal::evaluator<Derived>(derived()).template packet<LoadMode, DefaultPacketType>(index);
+  }
+
+ protected:
+  // explanation: DenseBase is doing "using ..." on the methods from DenseCoeffsBase.
+  // But some methods are only available in the DirectAccess case.
+  // So we add dummy methods here with these names, so that "using... " doesn't fail.
+  // It's not private so that the child class DenseBase can access them, and it's not public
+  // either since it's an implementation detail, so has to be protected.
+  void coeffRef();
+  void coeffRefByOuterInner();
+  void writePacket();
+  void writePacketByOuterInner();
+  void copyCoeff();
+  void copyCoeffByOuterInner();
+  void copyPacket();
+  void copyPacketByOuterInner();
+  void stride();
+  void innerStride();
+  void outerStride();
+  void rowStride();
+  void colStride();
+};
+
+/** \brief Base class providing read/write coefficient access to matrices and arrays.
+ * \ingroup Core_Module
+ * \tparam Derived Type of the derived class
+ *
+ * \note #WriteAccessors Constant indicating read/write access
+ *
+ * This class defines the non-const \c operator() function and friends, which can be used to write specific
+ * entries of a matrix or array. This class inherits DenseCoeffsBase<Derived, ReadOnlyAccessors> which
+ * defines the const variant for reading specific entries.
+ *
+ * \sa DenseCoeffsBase<Derived, DirectAccessors>, \ref TopicClassHierarchy
+ */
+template <typename Derived>
+class DenseCoeffsBase<Derived, WriteAccessors> : public DenseCoeffsBase<Derived, ReadOnlyAccessors> {
+ public:
+  typedef DenseCoeffsBase<Derived, ReadOnlyAccessors> Base;
+
+  typedef typename internal::traits<Derived>::StorageKind StorageKind;
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+  typedef typename internal::packet_traits<Scalar>::type PacketScalar;
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+
+  using Base::coeff;
+  using Base::colIndexByOuterInner;
+  using Base::cols;
+  using Base::derived;
+  using Base::rowIndexByOuterInner;
+  using Base::rows;
+  using Base::size;
+  using Base::operator[];
+  using Base::operator();
+  using Base::w;
+  using Base::x;
+  using Base::y;
+  using Base::z;
+
+  /** Short version: don't use this function, use
+   * \link operator()(Index,Index) \endlink instead.
+   *
+   * Long version: this function is similar to
+   * \link operator()(Index,Index) \endlink, but without the assertion.
+   * Use this for limiting the performance cost of debugging code when doing
+   * repeated coefficient access. Only use this when it is guaranteed that the
+   * parameters \a row and \a col are in range.
+   *
+   * If EIGEN_INTERNAL_DEBUGGING is defined, an assertion will be made, making this
+   * function equivalent to \link operator()(Index,Index) \endlink.
+   *
+   * \sa operator()(Index,Index), coeff(Index, Index) const, coeffRef(Index)
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Scalar& coeffRef(Index row, Index col) {
+    eigen_internal_assert(row >= 0 && row < rows() && col >= 0 && col < cols());
+    return internal::evaluator<Derived>(derived()).coeffRef(row, col);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRefByOuterInner(Index outer, Index inner) {
+    return coeffRef(rowIndexByOuterInner(outer, inner), colIndexByOuterInner(outer, inner));
+  }
+
+  /** \returns a reference to the coefficient at given the given row and column.
+   *
+   * \sa operator[](Index)
+   */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Scalar& operator()(Index row, Index col) {
+    eigen_assert(row >= 0 && row < rows() && col >= 0 && col < cols());
+    return coeffRef(row, col);
+  }
+
+  /** Short version: don't use this function, use
+   * \link operator[](Index) \endlink instead.
+   *
+   * Long version: this function is similar to
+   * \link operator[](Index) \endlink, but without the assertion.
+   * Use this for limiting the performance cost of debugging code when doing
+   * repeated coefficient access. Only use this when it is guaranteed that the
+   * parameters \a row and \a col are in range.
+   *
+   * If EIGEN_INTERNAL_DEBUGGING is defined, an assertion will be made, making this
+   * function equivalent to \link operator[](Index) \endlink.
+   *
+   * \sa operator[](Index), coeff(Index) const, coeffRef(Index,Index)
+   */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Scalar& coeffRef(Index index) {
+    EIGEN_STATIC_ASSERT(internal::evaluator<Derived>::Flags & LinearAccessBit,
+                        THIS_COEFFICIENT_ACCESSOR_TAKING_ONE_ACCESS_IS_ONLY_FOR_EXPRESSIONS_ALLOWING_LINEAR_ACCESS)
+    eigen_internal_assert(index >= 0 && index < size());
+    return internal::evaluator<Derived>(derived()).coeffRef(index);
+  }
+
+  /** \returns a reference to the coefficient at given index.
+   *
+   * This method is allowed only for vector expressions, and for matrix expressions having the LinearAccessBit.
+   *
+   * \sa operator[](Index) const, operator()(Index,Index), x(), y(), z(), w()
+   */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Scalar& operator[](Index index) {
+    EIGEN_STATIC_ASSERT(Derived::IsVectorAtCompileTime,
+                        THE_BRACKET_OPERATOR_IS_ONLY_FOR_VECTORS__USE_THE_PARENTHESIS_OPERATOR_INSTEAD)
+    eigen_assert(index >= 0 && index < size());
+    return coeffRef(index);
+  }
+
+  /** \returns a reference to the coefficient at given index.
+   *
+   * This is synonymous to operator[](Index).
+   *
+   * This method is allowed only for vector expressions, and for matrix expressions having the LinearAccessBit.
+   *
+   * \sa operator[](Index) const, operator()(Index,Index), x(), y(), z(), w()
+   */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Scalar& operator()(Index index) {
+    eigen_assert(index >= 0 && index < size());
+    return coeffRef(index);
+  }
+
+  /** equivalent to operator[](0).  */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Scalar& x() { return (*this)[0]; }
+
+  /** equivalent to operator[](1).  */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Scalar& y() {
+    EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime == -1 || Derived::SizeAtCompileTime >= 2, OUT_OF_RANGE_ACCESS);
+    return (*this)[1];
+  }
+
+  /** equivalent to operator[](2).  */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Scalar& z() {
+    EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime == -1 || Derived::SizeAtCompileTime >= 3, OUT_OF_RANGE_ACCESS);
+    return (*this)[2];
+  }
+
+  /** equivalent to operator[](3).  */
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Scalar& w() {
+    EIGEN_STATIC_ASSERT(Derived::SizeAtCompileTime == -1 || Derived::SizeAtCompileTime >= 4, OUT_OF_RANGE_ACCESS);
+    return (*this)[3];
+  }
+};
+
+/** \brief Base class providing direct read-only coefficient access to matrices and arrays.
+ * \ingroup Core_Module
+ * \tparam Derived Type of the derived class
+ *
+ * \note #DirectAccessors Constant indicating direct access
+ *
+ * This class defines functions to work with strides which can be used to access entries directly. This class
+ * inherits DenseCoeffsBase<Derived, ReadOnlyAccessors> which defines functions to access entries read-only using
+ * \c operator() .
+ *
+ * \sa \blank \ref TopicClassHierarchy
+ */
+template <typename Derived>
+class DenseCoeffsBase<Derived, DirectAccessors> : public DenseCoeffsBase<Derived, ReadOnlyAccessors> {
+ public:
+  typedef DenseCoeffsBase<Derived, ReadOnlyAccessors> Base;
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+
+  using Base::cols;
+  using Base::derived;
+  using Base::rows;
+  using Base::size;
+
+  /** \returns the pointer increment between two consecutive elements within a slice in the inner direction.
+   *
+   * \sa outerStride(), rowStride(), colStride()
+   */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const { return derived().innerStride(); }
+
+  /** \returns the pointer increment between two consecutive inner slices (for example, between two consecutive columns
+   *          in a column-major matrix).
+   *
+   * \sa innerStride(), rowStride(), colStride()
+   */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const { return derived().outerStride(); }
+
+  // FIXME shall we remove it ?
+  EIGEN_CONSTEXPR inline Index stride() const { return Derived::IsVectorAtCompileTime ? innerStride() : outerStride(); }
+
+  /** \returns the pointer increment between two consecutive rows.
+   *
+   * \sa innerStride(), outerStride(), colStride()
+   */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rowStride() const {
+    return Derived::IsRowMajor ? outerStride() : innerStride();
+  }
+
+  /** \returns the pointer increment between two consecutive columns.
+   *
+   * \sa innerStride(), outerStride(), rowStride()
+   */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index colStride() const {
+    return Derived::IsRowMajor ? innerStride() : outerStride();
+  }
+};
+
+/** \brief Base class providing direct read/write coefficient access to matrices and arrays.
+ * \ingroup Core_Module
+ * \tparam Derived Type of the derived class
+ *
+ * \note #DirectWriteAccessors Constant indicating direct access
+ *
+ * This class defines functions to work with strides which can be used to access entries directly. This class
+ * inherits DenseCoeffsBase<Derived, WriteAccessors> which defines functions to access entries read/write using
+ * \c operator().
+ *
+ * \sa \blank \ref TopicClassHierarchy
+ */
+template <typename Derived>
+class DenseCoeffsBase<Derived, DirectWriteAccessors> : public DenseCoeffsBase<Derived, WriteAccessors> {
+ public:
+  typedef DenseCoeffsBase<Derived, WriteAccessors> Base;
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+
+  using Base::cols;
+  using Base::derived;
+  using Base::rows;
+  using Base::size;
+
+  /** \returns the pointer increment between two consecutive elements within a slice in the inner direction.
+   *
+   * \sa outerStride(), rowStride(), colStride()
+   */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return derived().innerStride(); }
+
+  /** \returns the pointer increment between two consecutive inner slices (for example, between two consecutive columns
+   *          in a column-major matrix).
+   *
+   * \sa innerStride(), rowStride(), colStride()
+   */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return derived().outerStride(); }
+
+  // FIXME shall we remove it ?
+  EIGEN_CONSTEXPR inline Index stride() const EIGEN_NOEXCEPT {
+    return Derived::IsVectorAtCompileTime ? innerStride() : outerStride();
+  }
+
+  /** \returns the pointer increment between two consecutive rows.
+   *
+   * \sa innerStride(), outerStride(), colStride()
+   */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rowStride() const EIGEN_NOEXCEPT {
+    return Derived::IsRowMajor ? outerStride() : innerStride();
+  }
+
+  /** \returns the pointer increment between two consecutive columns.
+   *
+   * \sa innerStride(), outerStride(), rowStride()
+   */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index colStride() const EIGEN_NOEXCEPT {
+    return Derived::IsRowMajor ? innerStride() : outerStride();
+  }
+};
+
+namespace internal {
+
+template <int Alignment, typename Derived, bool JustReturnZero>
+struct first_aligned_impl {
+  static EIGEN_CONSTEXPR inline Index run(const Derived&) EIGEN_NOEXCEPT { return 0; }
+};
+
+template <int Alignment, typename Derived>
+struct first_aligned_impl<Alignment, Derived, false> {
+  static inline Index run(const Derived& m) { return internal::first_aligned<Alignment>(m.data(), m.size()); }
+};
+
+/** \internal \returns the index of the first element of the array stored by \a m that is properly aligned with respect
+ * to \a Alignment for vectorization.
+ *
+ * \tparam Alignment requested alignment in Bytes.
+ *
+ * There is also the variant first_aligned(const Scalar*, Integer) defined in Memory.h. See it for more
+ * documentation.
+ */
+template <int Alignment, typename Derived>
+static inline Index first_aligned(const DenseBase<Derived>& m) {
+  enum { ReturnZero = (int(evaluator<Derived>::Alignment) >= Alignment) || !(Derived::Flags & DirectAccessBit) };
+  return first_aligned_impl<Alignment, Derived, ReturnZero>::run(m.derived());
+}
+
+template <typename Derived>
+static inline Index first_default_aligned(const DenseBase<Derived>& m) {
+  typedef typename Derived::Scalar Scalar;
+  typedef typename packet_traits<Scalar>::type DefaultPacketType;
+  return internal::first_aligned<int(unpacket_traits<DefaultPacketType>::alignment), Derived>(m);
+}
+
+template <typename Derived, bool HasDirectAccess = has_direct_access<Derived>::ret>
+struct inner_stride_at_compile_time {
+  enum { ret = traits<Derived>::InnerStrideAtCompileTime };
+};
+
+template <typename Derived>
+struct inner_stride_at_compile_time<Derived, false> {
+  enum { ret = 0 };
+};
+
+template <typename Derived, bool HasDirectAccess = has_direct_access<Derived>::ret>
+struct outer_stride_at_compile_time {
+  enum { ret = traits<Derived>::OuterStrideAtCompileTime };
+};
+
+template <typename Derived>
+struct outer_stride_at_compile_time<Derived, false> {
+  enum { ret = 0 };
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_DENSECOEFFSBASE_H

include/Eigen/src/Core/DenseStorage.h

@@ -0,0 +1,578 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2006-2009 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2010-2013 Hauke Heibel <hauke.heibel@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_MATRIXSTORAGE_H
+#define EIGEN_MATRIXSTORAGE_H
+
+#ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN
+#define EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(X) \
+  X;                                                \
+  EIGEN_DENSE_STORAGE_CTOR_PLUGIN;
+#else
+#define EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(X)
+#endif
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+#if defined(EIGEN_DISABLE_UNALIGNED_ARRAY_ASSERT)
+#define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(Alignment)
+#else
+#define EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(Alignment)                                        \
+  eigen_assert((is_constant_evaluated() || (std::uintptr_t(array) % Alignment == 0)) &&     \
+               "this assertion is explained here: "                                         \
+               "http://eigen.tuxfamily.org/dox-devel/group__TopicUnalignedArrayAssert.html" \
+               " **** READ THIS WEB PAGE !!! ****");
+#endif
+
+#if EIGEN_STACK_ALLOCATION_LIMIT
+#define EIGEN_MAKE_STACK_ALLOCATION_ASSERT(X) \
+  EIGEN_STATIC_ASSERT(X <= EIGEN_STACK_ALLOCATION_LIMIT, OBJECT_ALLOCATED_ON_STACK_IS_TOO_BIG)
+#else
+#define EIGEN_MAKE_STACK_ALLOCATION_ASSERT(X)
+#endif
+
+/** \internal
+ * Static array. If the MatrixOrArrayOptions require auto-alignment, the array will be automatically aligned:
+ * to 16 bytes boundary if the total size is a multiple of 16 bytes.
+ */
+
+template <typename T, int Size, int MatrixOrArrayOptions,
+          int Alignment = (MatrixOrArrayOptions & DontAlign) ? 0 : compute_default_alignment<T, Size>::value>
+struct plain_array {
+  EIGEN_ALIGN_TO_BOUNDARY(Alignment) T array[Size];
+#if defined(EIGEN_NO_DEBUG) || defined(EIGEN_TESTING_PLAINOBJECT_CTOR)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr plain_array() = default;
+#else
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr plain_array() {
+    EIGEN_MAKE_UNALIGNED_ARRAY_ASSERT(Alignment)
+    EIGEN_MAKE_STACK_ALLOCATION_ASSERT(Size * sizeof(T))
+  }
+#endif
+};
+
+template <typename T, int Size, int MatrixOrArrayOptions>
+struct plain_array<T, Size, MatrixOrArrayOptions, 0> {
+  T array[Size];
+#if defined(EIGEN_NO_DEBUG) || defined(EIGEN_TESTING_PLAINOBJECT_CTOR)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr plain_array() = default;
+#else
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr plain_array() { EIGEN_MAKE_STACK_ALLOCATION_ASSERT(Size * sizeof(T)) }
+#endif
+};
+
+template <typename T, int MatrixOrArrayOptions, int Alignment>
+struct plain_array<T, 0, MatrixOrArrayOptions, Alignment> {
+  T array[1];
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr plain_array() = default;
+};
+
+template <typename T, int Size, int Options, int Alignment>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap_plain_array(plain_array<T, Size, Options, Alignment>& a,
+                                                                      plain_array<T, Size, Options, Alignment>& b,
+                                                                      Index a_size, Index b_size) {
+  Index common_size = numext::mini(a_size, b_size);
+  std::swap_ranges(a.array, a.array + common_size, b.array);
+  if (a_size > b_size)
+    smart_copy(a.array + common_size, a.array + a_size, b.array + common_size);
+  else if (b_size > a_size)
+    smart_copy(b.array + common_size, b.array + b_size, a.array + common_size);
+}
+
+template <typename T, int Size, int Rows, int Cols, int Options>
+class DenseStorage_impl {
+  plain_array<T, Size, Options> m_data;
+
+ public:
+#ifndef EIGEN_DENSE_STORAGE_CTOR_PLUGIN
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl&) = default;
+#else
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = Size)
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl& other) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = Size)
+    smart_copy(other.m_data.array, other.m_data.array + Size, m_data.array);
+  }
+#endif
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index /*size*/, Index /*rows*/, Index /*cols*/) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl& other) {
+    numext::swap(m_data, other.m_data);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index /*size*/, Index /*rows*/,
+                                                                          Index /*cols*/) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index /*size*/, Index /*rows*/, Index /*cols*/) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return Rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return Cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return Rows * Cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return m_data.array; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return m_data.array; }
+};
+template <typename T, int Size, int Cols, int Options>
+class DenseStorage_impl<T, Size, Dynamic, Cols, Options> {
+  plain_array<T, Size, Options> m_data;
+  Index m_rows = 0;
+
+ public:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl& other)
+      : m_rows(other.m_rows) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = other.size())
+    smart_copy(other.m_data.array, other.m_data.array + other.size(), m_data.array);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index size, Index rows, Index /*cols*/)
+      : m_rows(rows) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({})
+    EIGEN_UNUSED_VARIABLE(size)
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl& other) {
+    smart_copy(other.m_data.array, other.m_data.array + other.size(), m_data.array);
+    m_rows = other.m_rows;
+    return *this;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl& other) {
+    swap_plain_array(m_data, other.m_data, size(), other.size());
+    numext::swap(m_rows, other.m_rows);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index /*size*/, Index rows, Index /*cols*/) {
+    m_rows = rows;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index /*size*/, Index rows, Index /*cols*/) {
+    m_rows = rows;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return m_rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return Cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return m_rows * Cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return m_data.array; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return m_data.array; }
+};
+template <typename T, int Size, int Rows, int Options>
+class DenseStorage_impl<T, Size, Rows, Dynamic, Options> {
+  plain_array<T, Size, Options> m_data;
+  Index m_cols = 0;
+
+ public:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl& other)
+      : m_cols(other.m_cols) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = other.size())
+    smart_copy(other.m_data.array, other.m_data.array + other.size(), m_data.array);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index size, Index /*rows*/, Index cols)
+      : m_cols(cols) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({})
+    EIGEN_UNUSED_VARIABLE(size)
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl& other) {
+    smart_copy(other.m_data.array, other.m_data.array + other.size(), m_data.array);
+    m_cols = other.m_cols;
+    return *this;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl& other) {
+    swap_plain_array(m_data, other.m_data, size(), other.size());
+    numext::swap(m_cols, other.m_cols);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index /*size*/, Index /*rows*/, Index cols) {
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index /*size*/, Index /*rows*/, Index cols) {
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return Rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return Rows * m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return m_data.array; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return m_data.array; }
+};
+template <typename T, int Size, int Options>
+class DenseStorage_impl<T, Size, Dynamic, Dynamic, Options> {
+  plain_array<T, Size, Options> m_data;
+  Index m_rows = 0;
+  Index m_cols = 0;
+
+ public:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl& other)
+      : m_rows(other.m_rows), m_cols(other.m_cols) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = other.size())
+    smart_copy(other.m_data.array, other.m_data.array + other.size(), m_data.array);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index size, Index rows, Index cols)
+      : m_rows(rows), m_cols(cols) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({})
+    EIGEN_UNUSED_VARIABLE(size)
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl& other) {
+    smart_copy(other.m_data.array, other.m_data.array + other.size(), m_data.array);
+    m_rows = other.m_rows;
+    m_cols = other.m_cols;
+    return *this;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl& other) {
+    swap_plain_array(m_data, other.m_data, size(), other.size());
+    numext::swap(m_rows, other.m_rows);
+    numext::swap(m_cols, other.m_cols);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index /*size*/, Index rows, Index cols) {
+    m_rows = rows;
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index /*size*/, Index rows, Index cols) {
+    m_rows = rows;
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return m_rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return m_rows * m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return m_data.array; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return m_data.array; }
+};
+// null matrix variants
+template <typename T, int Rows, int Cols, int Options>
+class DenseStorage_impl<T, 0, Rows, Cols, Options> {
+ public:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index /*size*/, Index /*rows*/, Index /*cols*/) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl&) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index /*size*/, Index /*rows*/,
+                                                                          Index /*cols*/) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index /*size*/, Index /*rows*/, Index /*cols*/) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return Rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return Cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return Rows * Cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return nullptr; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return nullptr; }
+};
+template <typename T, int Cols, int Options>
+class DenseStorage_impl<T, 0, Dynamic, Cols, Options> {
+  Index m_rows = 0;
+
+ public:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index /*size*/, Index rows, Index /*cols*/)
+      : m_rows(rows) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl& other) noexcept {
+    numext::swap(m_rows, other.m_rows);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index /*size*/, Index rows, Index /*cols*/) {
+    m_rows = rows;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index /*size*/, Index rows, Index /*cols*/) {
+    m_rows = rows;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return m_rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return Cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return m_rows * Cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return nullptr; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return nullptr; }
+};
+template <typename T, int Rows, int Options>
+class DenseStorage_impl<T, 0, Rows, Dynamic, Options> {
+  Index m_cols = 0;
+
+ public:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index /*size*/, Index /*rows*/, Index cols)
+      : m_cols(cols) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl& other) noexcept {
+    numext::swap(m_cols, other.m_cols);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index /*size*/, Index /*rows*/, Index cols) {
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index /*size*/, Index /*rows*/, Index cols) {
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return Rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return Rows * m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return nullptr; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return nullptr; }
+};
+template <typename T, int Options>
+class DenseStorage_impl<T, 0, Dynamic, Dynamic, Options> {
+  Index m_rows = 0;
+  Index m_cols = 0;
+
+ public:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index /*size*/, Index rows, Index cols)
+      : m_rows(rows), m_cols(cols) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl& other) noexcept {
+    numext::swap(m_rows, other.m_rows);
+    numext::swap(m_cols, other.m_cols);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index /*size*/, Index rows, Index cols) {
+    m_rows = rows;
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index /*size*/, Index rows, Index cols) {
+    m_rows = rows;
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return m_rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return m_rows * m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return nullptr; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return nullptr; }
+};
+// fixed-size matrix with dynamic memory allocation not currently supported
+template <typename T, int Rows, int Cols, int Options>
+class DenseStorage_impl<T, Dynamic, Rows, Cols, Options> {};
+// dynamic-sized variants
+template <typename T, int Cols, int Options>
+class DenseStorage_impl<T, Dynamic, Dynamic, Cols, Options> {
+  static constexpr bool Align = (Options & DontAlign) == 0;
+  T* m_data = nullptr;
+  Index m_rows = 0;
+
+ public:
+  static constexpr int Size = Dynamic;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl& other)
+      : m_data(conditional_aligned_new_auto<T, Align>(other.size())), m_rows(other.m_rows) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = other.size())
+    smart_copy(other.m_data, other.m_data + other.size(), m_data);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index size, Index rows, Index /*cols*/)
+      : m_data(conditional_aligned_new_auto<T, Align>(size)), m_rows(rows) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({})
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(DenseStorage_impl&& other) noexcept
+      : m_data(other.m_data), m_rows(other.m_rows) {
+    other.m_data = nullptr;
+    other.m_rows = 0;
+  }
+  EIGEN_DEVICE_FUNC ~DenseStorage_impl() { conditional_aligned_delete_auto<T, Align>(m_data, size()); }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl& other) {
+    resize(other.size(), other.rows(), other.cols());
+    smart_copy(other.m_data, other.m_data + other.size(), m_data);
+    return *this;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(DenseStorage_impl&& other) noexcept {
+    this->swap(other);
+    return *this;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl& other) noexcept {
+    numext::swap(m_data, other.m_data);
+    numext::swap(m_rows, other.m_rows);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index size, Index rows, Index /*cols*/) {
+    m_data = conditional_aligned_realloc_new_auto<T, Align>(m_data, size, this->size());
+    m_rows = rows;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index size, Index rows, Index /*cols*/) {
+    Index oldSize = this->size();
+    if (oldSize != size) {
+      conditional_aligned_delete_auto<T, Align>(m_data, oldSize);
+      m_data = conditional_aligned_new_auto<T, Align>(size);
+      EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({})
+    }
+    m_rows = rows;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return m_rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return Cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return m_rows * Cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return m_data; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return m_data; }
+};
+template <typename T, int Rows, int Options>
+class DenseStorage_impl<T, Dynamic, Rows, Dynamic, Options> {
+  static constexpr bool Align = (Options & DontAlign) == 0;
+  T* m_data = nullptr;
+  Index m_cols = 0;
+
+ public:
+  static constexpr int Size = Dynamic;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl& other)
+      : m_data(conditional_aligned_new_auto<T, Align>(other.size())), m_cols(other.m_cols) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = other.size())
+    smart_copy(other.m_data, other.m_data + other.size(), m_data);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index size, Index /*rows*/, Index cols)
+      : m_data(conditional_aligned_new_auto<T, Align>(size)), m_cols(cols) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({})
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(DenseStorage_impl&& other) noexcept
+      : m_data(other.m_data), m_cols(other.m_cols) {
+    other.m_data = nullptr;
+    other.m_cols = 0;
+  }
+  EIGEN_DEVICE_FUNC ~DenseStorage_impl() { conditional_aligned_delete_auto<T, Align>(m_data, size()); }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl& other) {
+    resize(other.size(), other.rows(), other.cols());
+    smart_copy(other.m_data, other.m_data + other.size(), m_data);
+    return *this;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(DenseStorage_impl&& other) noexcept {
+    this->swap(other);
+    return *this;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl& other) noexcept {
+    numext::swap(m_data, other.m_data);
+    numext::swap(m_cols, other.m_cols);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index size, Index /*rows*/, Index cols) {
+    m_data = conditional_aligned_realloc_new_auto<T, Align>(m_data, size, this->size());
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index size, Index /*rows*/, Index cols) {
+    Index oldSize = this->size();
+    if (oldSize != size) {
+      conditional_aligned_delete_auto<T, Align>(m_data, oldSize);
+      m_data = conditional_aligned_new_auto<T, Align>(size);
+      EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({})
+    }
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return Rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return Rows * m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return m_data; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return m_data; }
+};
+template <typename T, int Options>
+class DenseStorage_impl<T, Dynamic, Dynamic, Dynamic, Options> {
+  static constexpr bool Align = (Options & DontAlign) == 0;
+  T* m_data = nullptr;
+  Index m_rows = 0;
+  Index m_cols = 0;
+
+ public:
+  static constexpr int Size = Dynamic;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(const DenseStorage_impl& other)
+      : m_data(conditional_aligned_new_auto<T, Align>(other.size())), m_rows(other.m_rows), m_cols(other.m_cols) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN(Index size = other.size())
+    smart_copy(other.m_data, other.m_data + other.size(), m_data);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(Index size, Index rows, Index cols)
+      : m_data(conditional_aligned_new_auto<T, Align>(size)), m_rows(rows), m_cols(cols) {
+    EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({})
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl(DenseStorage_impl&& other) noexcept
+      : m_data(other.m_data), m_rows(other.m_rows), m_cols(other.m_cols) {
+    other.m_data = nullptr;
+    other.m_rows = 0;
+    other.m_cols = 0;
+  }
+  EIGEN_DEVICE_FUNC ~DenseStorage_impl() { conditional_aligned_delete_auto<T, Align>(m_data, size()); }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(const DenseStorage_impl& other) {
+    resize(other.size(), other.rows(), other.cols());
+    smart_copy(other.m_data, other.m_data + other.size(), m_data);
+    return *this;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage_impl& operator=(DenseStorage_impl&& other) noexcept {
+    this->swap(other);
+    return *this;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void swap(DenseStorage_impl& other) noexcept {
+    numext::swap(m_data, other.m_data);
+    numext::swap(m_rows, other.m_rows);
+    numext::swap(m_cols, other.m_cols);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void conservativeResize(Index size, Index rows, Index cols) {
+    m_data = conditional_aligned_realloc_new_auto<T, Align>(m_data, size, this->size());
+    m_rows = rows;
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void resize(Index size, Index rows, Index cols) {
+    Index oldSize = this->size();
+    if (oldSize != size) {
+      conditional_aligned_delete_auto<T, Align>(m_data, oldSize);
+      m_data = conditional_aligned_new_auto<T, Align>(size);
+      EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN({})
+    }
+    m_rows = rows;
+    m_cols = cols;
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index rows() const { return m_rows; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index cols() const { return m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Index size() const { return m_rows * m_cols; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return m_data; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return m_data; }
+};
+template <typename T, int Size, int Rows, int Cols>
+struct use_default_move {
+  static constexpr bool DynamicObject = Size == Dynamic;
+  static constexpr bool TrivialObject =
+      (!NumTraits<T>::RequireInitialization) && (Rows >= 0) && (Cols >= 0) && (Size == Rows * Cols);
+  static constexpr bool value = DynamicObject || TrivialObject;
+};
+}  // end namespace internal
+
+/** \internal
+ *
+ * \class DenseStorage_impl
+ * \ingroup Core_Module
+ *
+ * \brief Stores the data of a matrix
+ *
+ * This class stores the data of fixed-size, dynamic-size or mixed matrices
+ * in a way as compact as possible.
+ *
+ * \sa Matrix
+ */
+template <typename T, int Size, int Rows, int Cols, int Options,
+          bool Trivial = internal::use_default_move<T, Size, Rows, Cols>::value>
+class DenseStorage : public internal::DenseStorage_impl<T, Size, Rows, Cols, Options> {
+  using Base = internal::DenseStorage_impl<T, Size, Rows, Cols, Options>;
+
+ public:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage(const DenseStorage&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage(Index size, Index rows, Index cols)
+      : Base(size, rows, cols) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage& operator=(const DenseStorage&) = default;
+  // if DenseStorage meets the requirements of use_default_move, then use the move construction and move assignment
+  // operation defined in DenseStorage_impl, or the compiler-generated version if none is defined
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage(DenseStorage&&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage& operator=(DenseStorage&&) = default;
+};
+template <typename T, int Size, int Rows, int Cols, int Options>
+class DenseStorage<T, Size, Rows, Cols, Options, false>
+    : public internal::DenseStorage_impl<T, Size, Rows, Cols, Options> {
+  using Base = internal::DenseStorage_impl<T, Size, Rows, Cols, Options>;
+
+ public:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage() = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage(const DenseStorage&) = default;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage(Index size, Index rows, Index cols)
+      : Base(size, rows, cols) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage& operator=(const DenseStorage&) = default;
+  // if DenseStorage does not meet the requirements of use_default_move, then defer to the copy construction and copy
+  // assignment behavior
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage(DenseStorage&& other)
+      : DenseStorage(static_cast<const DenseStorage&>(other)) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr DenseStorage& operator=(DenseStorage&& other) {
+    *this = other;
+    return *this;
+  }
+};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_MATRIX_H

include/Eigen/src/Core/DeviceWrapper.h

@@ -0,0 +1,155 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2023 Charlie Schlosser <cs.schlosser@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_DEVICEWRAPPER_H
+#define EIGEN_DEVICEWRAPPER_H
+
+namespace Eigen {
+template <typename Derived, typename Device>
+struct DeviceWrapper {
+  using Base = EigenBase<internal::remove_all_t<Derived>>;
+  using Scalar = typename Derived::Scalar;
+
+  EIGEN_DEVICE_FUNC DeviceWrapper(Base& xpr, Device& device) : m_xpr(xpr.derived()), m_device(device) {}
+  EIGEN_DEVICE_FUNC DeviceWrapper(const Base& xpr, Device& device) : m_xpr(xpr.derived()), m_device(device) {}
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const EigenBase<OtherDerived>& other) {
+    using AssignOp = internal::assign_op<Scalar, typename OtherDerived::Scalar>;
+    internal::call_assignment(*this, other.derived(), AssignOp());
+    return m_xpr;
+  }
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator+=(const EigenBase<OtherDerived>& other) {
+    using AddAssignOp = internal::add_assign_op<Scalar, typename OtherDerived::Scalar>;
+    internal::call_assignment(*this, other.derived(), AddAssignOp());
+    return m_xpr;
+  }
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator-=(const EigenBase<OtherDerived>& other) {
+    using SubAssignOp = internal::sub_assign_op<Scalar, typename OtherDerived::Scalar>;
+    internal::call_assignment(*this, other.derived(), SubAssignOp());
+    return m_xpr;
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& derived() { return m_xpr; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Device& device() { return m_device; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE NoAlias<DeviceWrapper, EigenBase> noalias() {
+    return NoAlias<DeviceWrapper, EigenBase>(*this);
+  }
+
+  Derived& m_xpr;
+  Device& m_device;
+};
+
+namespace internal {
+
+// this is where we differentiate between lazy assignment and specialized kernels (e.g. matrix products)
+template <typename DstXprType, typename SrcXprType, typename Functor, typename Device,
+          typename Kind = typename AssignmentKind<typename evaluator_traits<DstXprType>::Shape,
+                                                  typename evaluator_traits<SrcXprType>::Shape>::Kind,
+          typename EnableIf = void>
+struct AssignmentWithDevice;
+
+// unless otherwise specified, use the default product implementation
+template <typename DstXprType, typename Lhs, typename Rhs, int Options, typename Functor, typename Device,
+          typename Weak>
+struct AssignmentWithDevice<DstXprType, Product<Lhs, Rhs, Options>, Functor, Device, Dense2Dense, Weak> {
+  using SrcXprType = Product<Lhs, Rhs, Options>;
+  using Base = Assignment<DstXprType, SrcXprType, Functor>;
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src, const Functor& func,
+                                                        Device&) {
+    Base::run(dst, src, func);
+  };
+};
+
+// specialization for coeffcient-wise assignment
+template <typename DstXprType, typename SrcXprType, typename Functor, typename Device, typename Weak>
+struct AssignmentWithDevice<DstXprType, SrcXprType, Functor, Device, Dense2Dense, Weak> {
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src, const Functor& func,
+                                                        Device& device) {
+#ifndef EIGEN_NO_DEBUG
+    internal::check_for_aliasing(dst, src);
+#endif
+
+    call_dense_assignment_loop(dst, src, func, device);
+  }
+};
+
+// this allows us to use the default evaluation scheme if it is not specialized for the device
+template <typename Kernel, typename Device, int Traversal = Kernel::AssignmentTraits::Traversal,
+          int Unrolling = Kernel::AssignmentTraits::Unrolling>
+struct dense_assignment_loop_with_device {
+  using Base = dense_assignment_loop<Kernel, Traversal, Unrolling>;
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void run(Kernel& kernel, Device&) { Base::run(kernel); }
+};
+
+// entry point for a generic expression with device
+template <typename Dst, typename Src, typename Func, typename Device>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_assignment_no_alias(DeviceWrapper<Dst, Device> dst,
+                                                                                    const Src& src, const Func& func) {
+  enum {
+    NeedToTranspose = ((int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1) ||
+                       (int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)) &&
+                      int(Dst::SizeAtCompileTime) != 1
+  };
+
+  using ActualDstTypeCleaned = std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst>;
+  using ActualDstType = std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst&>;
+  ActualDstType actualDst(dst.derived());
+
+  // TODO check whether this is the right place to perform these checks:
+  EIGEN_STATIC_ASSERT_LVALUE(Dst)
+  EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned, Src)
+  EIGEN_CHECK_BINARY_COMPATIBILIY(Func, typename ActualDstTypeCleaned::Scalar, typename Src::Scalar);
+
+  // this provides a mechanism for specializing simple assignments, matrix products, etc
+  AssignmentWithDevice<ActualDstTypeCleaned, Src, Func, Device>::run(actualDst, src, func, dst.device());
+}
+
+// copy and pasted from AssignEvaluator except forward device to kernel
+template <typename DstXprType, typename SrcXprType, typename Functor, typename Device>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void call_dense_assignment_loop(DstXprType& dst,
+                                                                                      const SrcXprType& src,
+                                                                                      const Functor& func,
+                                                                                      Device& device) {
+  using DstEvaluatorType = evaluator<DstXprType>;
+  using SrcEvaluatorType = evaluator<SrcXprType>;
+
+  SrcEvaluatorType srcEvaluator(src);
+
+  // NOTE To properly handle A = (A*A.transpose())/s with A rectangular,
+  // we need to resize the destination after the source evaluator has been created.
+  resize_if_allowed(dst, src, func);
+
+  DstEvaluatorType dstEvaluator(dst);
+
+  using Kernel = generic_dense_assignment_kernel<DstEvaluatorType, SrcEvaluatorType, Functor>;
+
+  Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
+
+  dense_assignment_loop_with_device<Kernel, Device>::run(kernel, device);
+}
+
+}  // namespace internal
+
+template <typename Derived>
+template <typename Device>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DeviceWrapper<Derived, Device> EigenBase<Derived>::device(Device& device) {
+  return DeviceWrapper<Derived, Device>(derived(), device);
+}
+
+template <typename Derived>
+template <typename Device>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DeviceWrapper<const Derived, Device> EigenBase<Derived>::device(
+    Device& device) const {
+  return DeviceWrapper<const Derived, Device>(derived(), device);
+}
+}  // namespace Eigen
+#endif

include/Eigen/src/Core/Diagonal.h

@@ -0,0 +1,221 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2007-2009 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_DIAGONAL_H
+#define EIGEN_DIAGONAL_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class Diagonal
+ * \ingroup Core_Module
+ *
+ * \brief Expression of a diagonal/subdiagonal/superdiagonal in a matrix
+ *
+ * \tparam MatrixType the type of the object in which we are taking a sub/main/super diagonal
+ * \tparam DiagIndex the index of the sub/super diagonal. The default is 0 and it means the main diagonal.
+ *              A positive value means a superdiagonal, a negative value means a subdiagonal.
+ *              You can also use DynamicIndex so the index can be set at runtime.
+ *
+ * The matrix is not required to be square.
+ *
+ * This class represents an expression of the main diagonal, or any sub/super diagonal
+ * of a square matrix. It is the return type of MatrixBase::diagonal() and MatrixBase::diagonal(Index) and most of the
+ * time this is the only way it is used.
+ *
+ * \sa MatrixBase::diagonal(), MatrixBase::diagonal(Index)
+ */
+
+namespace internal {
+template <typename MatrixType, int DiagIndex>
+struct traits<Diagonal<MatrixType, DiagIndex> > : traits<MatrixType> {
+  typedef typename ref_selector<MatrixType>::type MatrixTypeNested;
+  typedef std::remove_reference_t<MatrixTypeNested> MatrixTypeNested_;
+  typedef typename MatrixType::StorageKind StorageKind;
+  enum {
+    RowsAtCompileTime = (int(DiagIndex) == DynamicIndex || int(MatrixType::SizeAtCompileTime) == Dynamic)
+                            ? Dynamic
+                            : (plain_enum_min(MatrixType::RowsAtCompileTime - plain_enum_max(-DiagIndex, 0),
+                                              MatrixType::ColsAtCompileTime - plain_enum_max(DiagIndex, 0))),
+    ColsAtCompileTime = 1,
+    MaxRowsAtCompileTime =
+        int(MatrixType::MaxSizeAtCompileTime) == Dynamic ? Dynamic
+        : DiagIndex == DynamicIndex
+            ? min_size_prefer_fixed(MatrixType::MaxRowsAtCompileTime, MatrixType::MaxColsAtCompileTime)
+            : (plain_enum_min(MatrixType::MaxRowsAtCompileTime - plain_enum_max(-DiagIndex, 0),
+                              MatrixType::MaxColsAtCompileTime - plain_enum_max(DiagIndex, 0))),
+    MaxColsAtCompileTime = 1,
+    MaskLvalueBit = is_lvalue<MatrixType>::value ? LvalueBit : 0,
+    Flags = (unsigned int)MatrixTypeNested_::Flags & (RowMajorBit | MaskLvalueBit | DirectAccessBit) &
+            ~RowMajorBit,  // FIXME DirectAccessBit should not be handled by expressions
+    MatrixTypeOuterStride = outer_stride_at_compile_time<MatrixType>::ret,
+    InnerStrideAtCompileTime = MatrixTypeOuterStride == Dynamic ? Dynamic : MatrixTypeOuterStride + 1,
+    OuterStrideAtCompileTime = 0
+  };
+};
+}  // namespace internal
+
+template <typename MatrixType, int DiagIndex_>
+class Diagonal : public internal::dense_xpr_base<Diagonal<MatrixType, DiagIndex_> >::type {
+ public:
+  enum { DiagIndex = DiagIndex_ };
+  typedef typename internal::dense_xpr_base<Diagonal>::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Diagonal)
+
+  EIGEN_DEVICE_FUNC explicit inline Diagonal(MatrixType& matrix, Index a_index = DiagIndex)
+      : m_matrix(matrix), m_index(a_index) {
+    eigen_assert(a_index <= m_matrix.cols() && -a_index <= m_matrix.rows());
+  }
+
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Diagonal)
+
+  EIGEN_DEVICE_FUNC inline Index rows() const {
+    return m_index.value() < 0 ? numext::mini<Index>(m_matrix.cols(), m_matrix.rows() + m_index.value())
+                               : numext::mini<Index>(m_matrix.rows(), m_matrix.cols() - m_index.value());
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return 1; }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT {
+    return m_matrix.outerStride() + 1;
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return 0; }
+
+  typedef std::conditional_t<internal::is_lvalue<MatrixType>::value, Scalar, const Scalar> ScalarWithConstIfNotLvalue;
+
+  EIGEN_DEVICE_FUNC inline ScalarWithConstIfNotLvalue* data() { return &(m_matrix.coeffRef(rowOffset(), colOffset())); }
+  EIGEN_DEVICE_FUNC inline const Scalar* data() const { return &(m_matrix.coeffRef(rowOffset(), colOffset())); }
+
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index row, Index) {
+    EIGEN_STATIC_ASSERT_LVALUE(MatrixType)
+    return m_matrix.coeffRef(row + rowOffset(), row + colOffset());
+  }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index row, Index) const {
+    return m_matrix.coeffRef(row + rowOffset(), row + colOffset());
+  }
+
+  EIGEN_DEVICE_FUNC inline CoeffReturnType coeff(Index row, Index) const {
+    return m_matrix.coeff(row + rowOffset(), row + colOffset());
+  }
+
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index idx) {
+    EIGEN_STATIC_ASSERT_LVALUE(MatrixType)
+    return m_matrix.coeffRef(idx + rowOffset(), idx + colOffset());
+  }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index idx) const {
+    return m_matrix.coeffRef(idx + rowOffset(), idx + colOffset());
+  }
+
+  EIGEN_DEVICE_FUNC inline CoeffReturnType coeff(Index idx) const {
+    return m_matrix.coeff(idx + rowOffset(), idx + colOffset());
+  }
+
+  EIGEN_DEVICE_FUNC inline const internal::remove_all_t<typename MatrixType::Nested>& nestedExpression() const {
+    return m_matrix;
+  }
+
+  EIGEN_DEVICE_FUNC inline Index index() const { return m_index.value(); }
+
+ protected:
+  typename internal::ref_selector<MatrixType>::non_const_type m_matrix;
+  const internal::variable_if_dynamicindex<Index, DiagIndex> m_index;
+
+ private:
+  // some compilers may fail to optimize std::max etc in case of compile-time constants...
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index absDiagIndex() const EIGEN_NOEXCEPT {
+    return m_index.value() > 0 ? m_index.value() : -m_index.value();
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rowOffset() const EIGEN_NOEXCEPT {
+    return m_index.value() > 0 ? 0 : -m_index.value();
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index colOffset() const EIGEN_NOEXCEPT {
+    return m_index.value() > 0 ? m_index.value() : 0;
+  }
+  // trigger a compile-time error if someone try to call packet
+  template <int LoadMode>
+  typename MatrixType::PacketReturnType packet(Index) const;
+  template <int LoadMode>
+  typename MatrixType::PacketReturnType packet(Index, Index) const;
+};
+
+/** \returns an expression of the main diagonal of the matrix \c *this
+ *
+ * \c *this is not required to be square.
+ *
+ * Example: \include MatrixBase_diagonal.cpp
+ * Output: \verbinclude MatrixBase_diagonal.out
+ *
+ * \sa class Diagonal */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline typename MatrixBase<Derived>::DiagonalReturnType MatrixBase<Derived>::diagonal() {
+  return DiagonalReturnType(derived());
+}
+
+/** This is the const version of diagonal(). */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline const typename MatrixBase<Derived>::ConstDiagonalReturnType MatrixBase<Derived>::diagonal()
+    const {
+  return ConstDiagonalReturnType(derived());
+}
+
+/** \returns an expression of the \a DiagIndex-th sub or super diagonal of the matrix \c *this
+ *
+ * \c *this is not required to be square.
+ *
+ * The template parameter \a DiagIndex represent a super diagonal if \a DiagIndex > 0
+ * and a sub diagonal otherwise. \a DiagIndex == 0 is equivalent to the main diagonal.
+ *
+ * Example: \include MatrixBase_diagonal_int.cpp
+ * Output: \verbinclude MatrixBase_diagonal_int.out
+ *
+ * \sa MatrixBase::diagonal(), class Diagonal */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline Diagonal<Derived, DynamicIndex> MatrixBase<Derived>::diagonal(Index index) {
+  return Diagonal<Derived, DynamicIndex>(derived(), index);
+}
+
+/** This is the const version of diagonal(Index). */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline const Diagonal<const Derived, DynamicIndex> MatrixBase<Derived>::diagonal(Index index) const {
+  return Diagonal<const Derived, DynamicIndex>(derived(), index);
+}
+
+/** \returns an expression of the \a DiagIndex-th sub or super diagonal of the matrix \c *this
+ *
+ * \c *this is not required to be square.
+ *
+ * The template parameter \a DiagIndex represent a super diagonal if \a DiagIndex > 0
+ * and a sub diagonal otherwise. \a DiagIndex == 0 is equivalent to the main diagonal.
+ *
+ * Example: \include MatrixBase_diagonal_template_int.cpp
+ * Output: \verbinclude MatrixBase_diagonal_template_int.out
+ *
+ * \sa MatrixBase::diagonal(), class Diagonal */
+template <typename Derived>
+template <int Index_>
+EIGEN_DEVICE_FUNC inline Diagonal<Derived, Index_> MatrixBase<Derived>::diagonal() {
+  return Diagonal<Derived, Index_>(derived());
+}
+
+/** This is the const version of diagonal<int>(). */
+template <typename Derived>
+template <int Index_>
+EIGEN_DEVICE_FUNC inline const Diagonal<const Derived, Index_> MatrixBase<Derived>::diagonal() const {
+  return Diagonal<const Derived, Index_>(derived());
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_DIAGONAL_H

include/Eigen/src/Core/DiagonalMatrix.h

@@ -0,0 +1,420 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2007-2009 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_DIAGONALMATRIX_H
+#define EIGEN_DIAGONALMATRIX_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class DiagonalBase
+ * \ingroup Core_Module
+ *
+ * \brief Base class for diagonal matrices and expressions
+ *
+ * This is the base class that is inherited by diagonal matrix and related expression
+ * types, which internally use a vector for storing the diagonal entries. Diagonal
+ * types always represent square matrices.
+ *
+ * \tparam Derived is the derived type, a DiagonalMatrix or DiagonalWrapper.
+ *
+ * \sa class DiagonalMatrix, class DiagonalWrapper
+ */
+template <typename Derived>
+class DiagonalBase : public EigenBase<Derived> {
+ public:
+  typedef typename internal::traits<Derived>::DiagonalVectorType DiagonalVectorType;
+  typedef typename DiagonalVectorType::Scalar Scalar;
+  typedef typename DiagonalVectorType::RealScalar RealScalar;
+  typedef typename internal::traits<Derived>::StorageKind StorageKind;
+  typedef typename internal::traits<Derived>::StorageIndex StorageIndex;
+
+  enum {
+    RowsAtCompileTime = DiagonalVectorType::SizeAtCompileTime,
+    ColsAtCompileTime = DiagonalVectorType::SizeAtCompileTime,
+    MaxRowsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime,
+    MaxColsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime,
+    IsVectorAtCompileTime = 0,
+    Flags = NoPreferredStorageOrderBit
+  };
+
+  typedef Matrix<Scalar, RowsAtCompileTime, ColsAtCompileTime, 0, MaxRowsAtCompileTime, MaxColsAtCompileTime>
+      DenseMatrixType;
+  typedef DenseMatrixType DenseType;
+  typedef DiagonalMatrix<Scalar, DiagonalVectorType::SizeAtCompileTime, DiagonalVectorType::MaxSizeAtCompileTime>
+      PlainObject;
+
+  /** \returns a reference to the derived object. */
+  EIGEN_DEVICE_FUNC inline const Derived& derived() const { return *static_cast<const Derived*>(this); }
+  /** \returns a const reference to the derived object. */
+  EIGEN_DEVICE_FUNC inline Derived& derived() { return *static_cast<Derived*>(this); }
+
+  /**
+   * Constructs a dense matrix from \c *this. Note, this directly returns a dense matrix type,
+   * not an expression.
+   * \returns A dense matrix, with its diagonal entries set from the the derived object. */
+  EIGEN_DEVICE_FUNC DenseMatrixType toDenseMatrix() const { return derived(); }
+
+  /** \returns a reference to the derived object's vector of diagonal coefficients. */
+  EIGEN_DEVICE_FUNC inline const DiagonalVectorType& diagonal() const { return derived().diagonal(); }
+  /** \returns a const reference to the derived object's vector of diagonal coefficients. */
+  EIGEN_DEVICE_FUNC inline DiagonalVectorType& diagonal() { return derived().diagonal(); }
+
+  /** \returns the value of the coefficient as if \c *this was a dense matrix. */
+  EIGEN_DEVICE_FUNC inline Scalar coeff(Index row, Index col) const {
+    eigen_assert(row >= 0 && col >= 0 && row < rows() && col <= cols());
+    return row == col ? diagonal().coeff(row) : Scalar(0);
+  }
+
+  /** \returns the number of rows. */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const { return diagonal().size(); }
+  /** \returns the number of columns. */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const { return diagonal().size(); }
+
+  /** \returns the diagonal matrix product of \c *this by the dense matrix, \a matrix */
+  template <typename MatrixDerived>
+  EIGEN_DEVICE_FUNC const Product<Derived, MatrixDerived, LazyProduct> operator*(
+      const MatrixBase<MatrixDerived>& matrix) const {
+    return Product<Derived, MatrixDerived, LazyProduct>(derived(), matrix.derived());
+  }
+
+  template <typename OtherDerived>
+  using DiagonalProductReturnType = DiagonalWrapper<const EIGEN_CWISE_BINARY_RETURN_TYPE(
+      DiagonalVectorType, typename OtherDerived::DiagonalVectorType, product)>;
+
+  /** \returns the diagonal matrix product of \c *this by the diagonal matrix \a other */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC const DiagonalProductReturnType<OtherDerived> operator*(
+      const DiagonalBase<OtherDerived>& other) const {
+    return diagonal().cwiseProduct(other.diagonal()).asDiagonal();
+  }
+
+  using DiagonalInverseReturnType =
+      DiagonalWrapper<const CwiseUnaryOp<internal::scalar_inverse_op<Scalar>, const DiagonalVectorType>>;
+
+  /** \returns the inverse \c *this. Computed as the coefficient-wise inverse of the diagonal. */
+  EIGEN_DEVICE_FUNC inline const DiagonalInverseReturnType inverse() const {
+    return diagonal().cwiseInverse().asDiagonal();
+  }
+
+  using DiagonalScaleReturnType =
+      DiagonalWrapper<const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(DiagonalVectorType, Scalar, product)>;
+
+  /** \returns the product of \c *this by the scalar \a scalar */
+  EIGEN_DEVICE_FUNC inline const DiagonalScaleReturnType operator*(const Scalar& scalar) const {
+    return (diagonal() * scalar).asDiagonal();
+  }
+
+  using ScaleDiagonalReturnType =
+      DiagonalWrapper<const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(Scalar, DiagonalVectorType, product)>;
+
+  /** \returns the product of a scalar and the diagonal matrix \a other */
+  EIGEN_DEVICE_FUNC friend inline const ScaleDiagonalReturnType operator*(const Scalar& scalar,
+                                                                          const DiagonalBase& other) {
+    return (scalar * other.diagonal()).asDiagonal();
+  }
+
+  template <typename OtherDerived>
+  using DiagonalSumReturnType = DiagonalWrapper<const EIGEN_CWISE_BINARY_RETURN_TYPE(
+      DiagonalVectorType, typename OtherDerived::DiagonalVectorType, sum)>;
+
+  /** \returns the sum of \c *this and the diagonal matrix \a other */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline const DiagonalSumReturnType<OtherDerived> operator+(
+      const DiagonalBase<OtherDerived>& other) const {
+    return (diagonal() + other.diagonal()).asDiagonal();
+  }
+
+  template <typename OtherDerived>
+  using DiagonalDifferenceReturnType = DiagonalWrapper<const EIGEN_CWISE_BINARY_RETURN_TYPE(
+      DiagonalVectorType, typename OtherDerived::DiagonalVectorType, difference)>;
+
+  /** \returns the difference of \c *this and the diagonal matrix \a other */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline const DiagonalDifferenceReturnType<OtherDerived> operator-(
+      const DiagonalBase<OtherDerived>& other) const {
+    return (diagonal() - other.diagonal()).asDiagonal();
+  }
+};
+
+/** \class DiagonalMatrix
+ * \ingroup Core_Module
+ *
+ * \brief Represents a diagonal matrix with its storage
+ *
+ * \tparam Scalar_ the type of coefficients
+ * \tparam SizeAtCompileTime the dimension of the matrix, or Dynamic
+ * \tparam MaxSizeAtCompileTime the dimension of the matrix, or Dynamic. This parameter is optional and defaults
+ *        to SizeAtCompileTime. Most of the time, you do not need to specify it.
+ *
+ * \sa class DiagonalBase, class DiagonalWrapper
+ */
+
+namespace internal {
+template <typename Scalar_, int SizeAtCompileTime, int MaxSizeAtCompileTime>
+struct traits<DiagonalMatrix<Scalar_, SizeAtCompileTime, MaxSizeAtCompileTime>>
+    : traits<Matrix<Scalar_, SizeAtCompileTime, SizeAtCompileTime, 0, MaxSizeAtCompileTime, MaxSizeAtCompileTime>> {
+  typedef Matrix<Scalar_, SizeAtCompileTime, 1, 0, MaxSizeAtCompileTime, 1> DiagonalVectorType;
+  typedef DiagonalShape StorageKind;
+  enum { Flags = LvalueBit | NoPreferredStorageOrderBit | NestByRefBit };
+};
+}  // namespace internal
+template <typename Scalar_, int SizeAtCompileTime, int MaxSizeAtCompileTime>
+class DiagonalMatrix : public DiagonalBase<DiagonalMatrix<Scalar_, SizeAtCompileTime, MaxSizeAtCompileTime>> {
+ public:
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef typename internal::traits<DiagonalMatrix>::DiagonalVectorType DiagonalVectorType;
+  typedef const DiagonalMatrix& Nested;
+  typedef Scalar_ Scalar;
+  typedef typename internal::traits<DiagonalMatrix>::StorageKind StorageKind;
+  typedef typename internal::traits<DiagonalMatrix>::StorageIndex StorageIndex;
+#endif
+
+ protected:
+  DiagonalVectorType m_diagonal;
+
+ public:
+  /** const version of diagonal(). */
+  EIGEN_DEVICE_FUNC inline const DiagonalVectorType& diagonal() const { return m_diagonal; }
+  /** \returns a reference to the stored vector of diagonal coefficients. */
+  EIGEN_DEVICE_FUNC inline DiagonalVectorType& diagonal() { return m_diagonal; }
+
+  /** Default constructor without initialization */
+  EIGEN_DEVICE_FUNC inline DiagonalMatrix() {}
+
+  /** Constructs a diagonal matrix with given dimension  */
+  EIGEN_DEVICE_FUNC explicit inline DiagonalMatrix(Index dim) : m_diagonal(dim) {}
+
+  /** 2D constructor. */
+  EIGEN_DEVICE_FUNC inline DiagonalMatrix(const Scalar& x, const Scalar& y) : m_diagonal(x, y) {}
+
+  /** 3D constructor. */
+  EIGEN_DEVICE_FUNC inline DiagonalMatrix(const Scalar& x, const Scalar& y, const Scalar& z) : m_diagonal(x, y, z) {}
+
+  /** \brief Construct a diagonal matrix with fixed size from an arbitrary number of coefficients.
+   *
+   * \warning To construct a diagonal matrix of fixed size, the number of values passed to this
+   * constructor must match the fixed dimension of \c *this.
+   *
+   * \sa DiagonalMatrix(const Scalar&, const Scalar&)
+   * \sa DiagonalMatrix(const Scalar&, const Scalar&, const Scalar&)
+   */
+  template <typename... ArgTypes>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DiagonalMatrix(const Scalar& a0, const Scalar& a1, const Scalar& a2,
+                                                       const ArgTypes&... args)
+      : m_diagonal(a0, a1, a2, args...) {}
+
+  /** \brief Constructs a DiagonalMatrix and initializes it by elements given by an initializer list of initializer
+   * lists \cpp11
+   */
+  EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE DiagonalMatrix(
+      const std::initializer_list<std::initializer_list<Scalar>>& list)
+      : m_diagonal(list) {}
+
+  /** \brief Constructs a DiagonalMatrix from an r-value diagonal vector type */
+  EIGEN_DEVICE_FUNC explicit inline DiagonalMatrix(DiagonalVectorType&& diag) : m_diagonal(std::move(diag)) {}
+
+  /** Copy constructor. */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline DiagonalMatrix(const DiagonalBase<OtherDerived>& other) : m_diagonal(other.diagonal()) {}
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** copy constructor. prevent a default copy constructor from hiding the other templated constructor */
+  inline DiagonalMatrix(const DiagonalMatrix& other) : m_diagonal(other.diagonal()) {}
+#endif
+
+  /** generic constructor from expression of the diagonal coefficients */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC explicit inline DiagonalMatrix(const MatrixBase<OtherDerived>& other) : m_diagonal(other) {}
+
+  /** Copy operator. */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC DiagonalMatrix& operator=(const DiagonalBase<OtherDerived>& other) {
+    m_diagonal = other.diagonal();
+    return *this;
+  }
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** This is a special case of the templated operator=. Its purpose is to
+   * prevent a default operator= from hiding the templated operator=.
+   */
+  EIGEN_DEVICE_FUNC DiagonalMatrix& operator=(const DiagonalMatrix& other) {
+    m_diagonal = other.diagonal();
+    return *this;
+  }
+#endif
+
+  typedef DiagonalWrapper<const CwiseNullaryOp<internal::scalar_constant_op<Scalar>, DiagonalVectorType>>
+      InitializeReturnType;
+
+  typedef DiagonalWrapper<const CwiseNullaryOp<internal::scalar_zero_op<Scalar>, DiagonalVectorType>>
+      ZeroInitializeReturnType;
+
+  /** Initializes a diagonal matrix of size SizeAtCompileTime with coefficients set to zero */
+  EIGEN_DEVICE_FUNC static const ZeroInitializeReturnType Zero() { return DiagonalVectorType::Zero().asDiagonal(); }
+  /** Initializes a diagonal matrix of size dim with coefficients set to zero */
+  EIGEN_DEVICE_FUNC static const ZeroInitializeReturnType Zero(Index size) {
+    return DiagonalVectorType::Zero(size).asDiagonal();
+  }
+  /** Initializes a identity matrix of size SizeAtCompileTime */
+  EIGEN_DEVICE_FUNC static const InitializeReturnType Identity() { return DiagonalVectorType::Ones().asDiagonal(); }
+  /** Initializes a identity matrix of size dim */
+  EIGEN_DEVICE_FUNC static const InitializeReturnType Identity(Index size) {
+    return DiagonalVectorType::Ones(size).asDiagonal();
+  }
+
+  /** Resizes to given size. */
+  EIGEN_DEVICE_FUNC inline void resize(Index size) { m_diagonal.resize(size); }
+  /** Sets all coefficients to zero. */
+  EIGEN_DEVICE_FUNC inline void setZero() { m_diagonal.setZero(); }
+  /** Resizes and sets all coefficients to zero. */
+  EIGEN_DEVICE_FUNC inline void setZero(Index size) { m_diagonal.setZero(size); }
+  /** Sets this matrix to be the identity matrix of the current size. */
+  EIGEN_DEVICE_FUNC inline void setIdentity() { m_diagonal.setOnes(); }
+  /** Sets this matrix to be the identity matrix of the given size. */
+  EIGEN_DEVICE_FUNC inline void setIdentity(Index size) { m_diagonal.setOnes(size); }
+};
+
+/** \class DiagonalWrapper
+ * \ingroup Core_Module
+ *
+ * \brief Expression of a diagonal matrix
+ *
+ * \tparam DiagonalVectorType_ the type of the vector of diagonal coefficients
+ *
+ * This class is an expression of a diagonal matrix, but not storing its own vector of diagonal coefficients,
+ * instead wrapping an existing vector expression. It is the return type of MatrixBase::asDiagonal()
+ * and most of the time this is the only way that it is used.
+ *
+ * \sa class DiagonalMatrix, class DiagonalBase, MatrixBase::asDiagonal()
+ */
+
+namespace internal {
+template <typename DiagonalVectorType_>
+struct traits<DiagonalWrapper<DiagonalVectorType_>> {
+  typedef DiagonalVectorType_ DiagonalVectorType;
+  typedef typename DiagonalVectorType::Scalar Scalar;
+  typedef typename DiagonalVectorType::StorageIndex StorageIndex;
+  typedef DiagonalShape StorageKind;
+  typedef typename traits<DiagonalVectorType>::XprKind XprKind;
+  enum {
+    RowsAtCompileTime = DiagonalVectorType::SizeAtCompileTime,
+    ColsAtCompileTime = DiagonalVectorType::SizeAtCompileTime,
+    MaxRowsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime,
+    MaxColsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime,
+    Flags = (traits<DiagonalVectorType>::Flags & LvalueBit) | NoPreferredStorageOrderBit
+  };
+};
+}  // namespace internal
+
+template <typename DiagonalVectorType_>
+class DiagonalWrapper : public DiagonalBase<DiagonalWrapper<DiagonalVectorType_>>, internal::no_assignment_operator {
+ public:
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef DiagonalVectorType_ DiagonalVectorType;
+  typedef DiagonalWrapper Nested;
+#endif
+
+  /** Constructor from expression of diagonal coefficients to wrap. */
+  EIGEN_DEVICE_FUNC explicit inline DiagonalWrapper(DiagonalVectorType& a_diagonal) : m_diagonal(a_diagonal) {}
+
+  /** \returns a const reference to the wrapped expression of diagonal coefficients. */
+  EIGEN_DEVICE_FUNC const DiagonalVectorType& diagonal() const { return m_diagonal; }
+
+ protected:
+  typename DiagonalVectorType::Nested m_diagonal;
+};
+
+/** \returns a pseudo-expression of a diagonal matrix with *this as vector of diagonal coefficients
+ *
+ * \only_for_vectors
+ *
+ * Example: \include MatrixBase_asDiagonal.cpp
+ * Output: \verbinclude MatrixBase_asDiagonal.out
+ *
+ * \sa class DiagonalWrapper, class DiagonalMatrix, diagonal(), isDiagonal()
+ **/
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline const DiagonalWrapper<const Derived> MatrixBase<Derived>::asDiagonal() const {
+  return DiagonalWrapper<const Derived>(derived());
+}
+
+/** \returns true if *this is approximately equal to a diagonal matrix,
+ *          within the precision given by \a prec.
+ *
+ * Example: \include MatrixBase_isDiagonal.cpp
+ * Output: \verbinclude MatrixBase_isDiagonal.out
+ *
+ * \sa asDiagonal()
+ */
+template <typename Derived>
+bool MatrixBase<Derived>::isDiagonal(const RealScalar& prec) const {
+  if (cols() != rows()) return false;
+  RealScalar maxAbsOnDiagonal = static_cast<RealScalar>(-1);
+  for (Index j = 0; j < cols(); ++j) {
+    RealScalar absOnDiagonal = numext::abs(coeff(j, j));
+    if (absOnDiagonal > maxAbsOnDiagonal) maxAbsOnDiagonal = absOnDiagonal;
+  }
+  for (Index j = 0; j < cols(); ++j)
+    for (Index i = 0; i < j; ++i) {
+      if (!internal::isMuchSmallerThan(coeff(i, j), maxAbsOnDiagonal, prec)) return false;
+      if (!internal::isMuchSmallerThan(coeff(j, i), maxAbsOnDiagonal, prec)) return false;
+    }
+  return true;
+}
+
+namespace internal {
+
+template <>
+struct storage_kind_to_shape<DiagonalShape> {
+  typedef DiagonalShape Shape;
+};
+
+struct Diagonal2Dense {};
+
+template <>
+struct AssignmentKind<DenseShape, DiagonalShape> {
+  typedef Diagonal2Dense Kind;
+};
+
+// Diagonal matrix to Dense assignment
+template <typename DstXprType, typename SrcXprType, typename Functor>
+struct Assignment<DstXprType, SrcXprType, Functor, Diagonal2Dense> {
+  static EIGEN_DEVICE_FUNC void run(
+      DstXprType& dst, const SrcXprType& src,
+      const internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>& /*func*/) {
+    Index dstRows = src.rows();
+    Index dstCols = src.cols();
+    if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
+
+    dst.setZero();
+    dst.diagonal() = src.diagonal();
+  }
+
+  static EIGEN_DEVICE_FUNC void run(
+      DstXprType& dst, const SrcXprType& src,
+      const internal::add_assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>& /*func*/) {
+    dst.diagonal() += src.diagonal();
+  }
+
+  static EIGEN_DEVICE_FUNC void run(
+      DstXprType& dst, const SrcXprType& src,
+      const internal::sub_assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>& /*func*/) {
+    dst.diagonal() -= src.diagonal();
+  }
+};
+
+}  // namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_DIAGONALMATRIX_H

include/Eigen/src/Core/DiagonalProduct.h

@@ -0,0 +1,30 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2007-2009 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_DIAGONALPRODUCT_H
+#define EIGEN_DIAGONALPRODUCT_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \returns the diagonal matrix product of \c *this by the diagonal matrix \a diagonal.
+ */
+template <typename Derived>
+template <typename DiagonalDerived>
+EIGEN_DEVICE_FUNC inline const Product<Derived, DiagonalDerived, LazyProduct> MatrixBase<Derived>::operator*(
+    const DiagonalBase<DiagonalDerived> &a_diagonal) const {
+  return Product<Derived, DiagonalDerived, LazyProduct>(derived(), a_diagonal.derived());
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_DIAGONALPRODUCT_H

include/Eigen/src/Core/Dot.h

@@ -0,0 +1,268 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2008, 2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_DOT_H
+#define EIGEN_DOT_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Derived, typename Scalar = typename traits<Derived>::Scalar>
+struct squared_norm_impl {
+  using Real = typename NumTraits<Scalar>::Real;
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Real run(const Derived& a) {
+    Scalar result = a.unaryExpr(squared_norm_functor<Scalar>()).sum();
+    return numext::real(result) + numext::imag(result);
+  }
+};
+
+template <typename Derived>
+struct squared_norm_impl<Derived, bool> {
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const Derived& a) { return a.any(); }
+};
+
+}  // end namespace internal
+
+/** \fn MatrixBase::dot
+ * \returns the dot product of *this with other.
+ *
+ * \only_for_vectors
+ *
+ * \note If the scalar type is complex numbers, then this function returns the hermitian
+ * (sesquilinear) dot product, conjugate-linear in the first variable and linear in the
+ * second variable.
+ *
+ * \sa squaredNorm(), norm()
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
+    typename ScalarBinaryOpTraits<typename internal::traits<Derived>::Scalar,
+                                  typename internal::traits<OtherDerived>::Scalar>::ReturnType
+    MatrixBase<Derived>::dot(const MatrixBase<OtherDerived>& other) const {
+  return internal::dot_impl<Derived, OtherDerived>::run(derived(), other.derived());
+}
+
+//---------- implementation of L2 norm and related functions ----------
+
+/** \returns, for vectors, the squared \em l2 norm of \c *this, and for matrices the squared Frobenius norm.
+ * In both cases, it consists in the sum of the square of all the matrix entries.
+ * For vectors, this is also equals to the dot product of \c *this with itself.
+ *
+ * \sa dot(), norm(), lpNorm()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename NumTraits<typename internal::traits<Derived>::Scalar>::Real
+MatrixBase<Derived>::squaredNorm() const {
+  return internal::squared_norm_impl<Derived>::run(derived());
+}
+
+/** \returns, for vectors, the \em l2 norm of \c *this, and for matrices the Frobenius norm.
+ * In both cases, it consists in the square root of the sum of the square of all the matrix entries.
+ * For vectors, this is also equals to the square root of the dot product of \c *this with itself.
+ *
+ * \sa lpNorm(), dot(), squaredNorm()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename NumTraits<typename internal::traits<Derived>::Scalar>::Real
+MatrixBase<Derived>::norm() const {
+  return numext::sqrt(squaredNorm());
+}
+
+/** \returns an expression of the quotient of \c *this by its own norm.
+ *
+ * \warning If the input vector is too small (i.e., this->norm()==0),
+ *          then this function returns a copy of the input.
+ *
+ * \only_for_vectors
+ *
+ * \sa norm(), normalize()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::PlainObject MatrixBase<Derived>::normalized()
+    const {
+  typedef typename internal::nested_eval<Derived, 2>::type Nested_;
+  Nested_ n(derived());
+  RealScalar z = n.squaredNorm();
+  // NOTE: after extensive benchmarking, this conditional does not impact performance, at least on recent x86 CPU
+  if (z > RealScalar(0))
+    return n / numext::sqrt(z);
+  else
+    return n;
+}
+
+/** Normalizes the vector, i.e. divides it by its own norm.
+ *
+ * \only_for_vectors
+ *
+ * \warning If the input vector is too small (i.e., this->norm()==0), then \c *this is left unchanged.
+ *
+ * \sa norm(), normalized()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void MatrixBase<Derived>::normalize() {
+  RealScalar z = squaredNorm();
+  // NOTE: after extensive benchmarking, this conditional does not impact performance, at least on recent x86 CPU
+  if (z > RealScalar(0)) derived() /= numext::sqrt(z);
+}
+
+/** \returns an expression of the quotient of \c *this by its own norm while avoiding underflow and overflow.
+ *
+ * \only_for_vectors
+ *
+ * This method is analogue to the normalized() method, but it reduces the risk of
+ * underflow and overflow when computing the norm.
+ *
+ * \warning If the input vector is too small (i.e., this->norm()==0),
+ *          then this function returns a copy of the input.
+ *
+ * \sa stableNorm(), stableNormalize(), normalized()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename MatrixBase<Derived>::PlainObject
+MatrixBase<Derived>::stableNormalized() const {
+  typedef typename internal::nested_eval<Derived, 3>::type Nested_;
+  Nested_ n(derived());
+  RealScalar w = n.cwiseAbs().maxCoeff();
+  RealScalar z = (n / w).squaredNorm();
+  if (z > RealScalar(0))
+    return n / (numext::sqrt(z) * w);
+  else
+    return n;
+}
+
+/** Normalizes the vector while avoid underflow and overflow
+ *
+ * \only_for_vectors
+ *
+ * This method is analogue to the normalize() method, but it reduces the risk of
+ * underflow and overflow when computing the norm.
+ *
+ * \warning If the input vector is too small (i.e., this->norm()==0), then \c *this is left unchanged.
+ *
+ * \sa stableNorm(), stableNormalized(), normalize()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void MatrixBase<Derived>::stableNormalize() {
+  RealScalar w = cwiseAbs().maxCoeff();
+  RealScalar z = (derived() / w).squaredNorm();
+  if (z > RealScalar(0)) derived() /= numext::sqrt(z) * w;
+}
+
+//---------- implementation of other norms ----------
+
+namespace internal {
+
+template <typename Derived, int p>
+struct lpNorm_selector {
+  typedef typename NumTraits<typename traits<Derived>::Scalar>::Real RealScalar;
+  EIGEN_DEVICE_FUNC static inline RealScalar run(const MatrixBase<Derived>& m) {
+    EIGEN_USING_STD(pow)
+    return pow(m.cwiseAbs().array().pow(p).sum(), RealScalar(1) / p);
+  }
+};
+
+template <typename Derived>
+struct lpNorm_selector<Derived, 1> {
+  EIGEN_DEVICE_FUNC static inline typename NumTraits<typename traits<Derived>::Scalar>::Real run(
+      const MatrixBase<Derived>& m) {
+    return m.cwiseAbs().sum();
+  }
+};
+
+template <typename Derived>
+struct lpNorm_selector<Derived, 2> {
+  EIGEN_DEVICE_FUNC static inline typename NumTraits<typename traits<Derived>::Scalar>::Real run(
+      const MatrixBase<Derived>& m) {
+    return m.norm();
+  }
+};
+
+template <typename Derived>
+struct lpNorm_selector<Derived, Infinity> {
+  typedef typename NumTraits<typename traits<Derived>::Scalar>::Real RealScalar;
+  EIGEN_DEVICE_FUNC static inline RealScalar run(const MatrixBase<Derived>& m) {
+    if (Derived::SizeAtCompileTime == 0 || (Derived::SizeAtCompileTime == Dynamic && m.size() == 0))
+      return RealScalar(0);
+    return m.cwiseAbs().maxCoeff();
+  }
+};
+
+}  // end namespace internal
+
+/** \returns the \b coefficient-wise \f$ \ell^p \f$ norm of \c *this, that is, returns the p-th root of the sum of the
+ * p-th powers of the absolute values of the coefficients of \c *this. If \a p is the special value \a Eigen::Infinity,
+ * this function returns the \f$ \ell^\infty \f$ norm, that is the maximum of the absolute values of the coefficients of
+ * \c *this.
+ *
+ * In all cases, if \c *this is empty, then the value 0 is returned.
+ *
+ * \note For matrices, this function does not compute the <a
+ * href="https://en.wikipedia.org/wiki/Operator_norm">operator-norm</a>. That is, if \c *this is a matrix, then its
+ * coefficients are interpreted as a 1D vector. Nonetheless, you can easily compute the 1-norm and \f$\infty\f$-norm
+ * matrix operator norms using \link TutorialReductionsVisitorsBroadcastingReductionsNorm partial reductions \endlink.
+ *
+ * \sa norm()
+ */
+template <typename Derived>
+template <int p>
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+EIGEN_DEVICE_FUNC inline typename NumTraits<typename internal::traits<Derived>::Scalar>::Real
+#else
+EIGEN_DEVICE_FUNC MatrixBase<Derived>::RealScalar
+#endif
+MatrixBase<Derived>::lpNorm() const {
+  return internal::lpNorm_selector<Derived, p>::run(*this);
+}
+
+//---------- implementation of isOrthogonal / isUnitary ----------
+
+/** \returns true if *this is approximately orthogonal to \a other,
+ *          within the precision given by \a prec.
+ *
+ * Example: \include MatrixBase_isOrthogonal.cpp
+ * Output: \verbinclude MatrixBase_isOrthogonal.out
+ */
+template <typename Derived>
+template <typename OtherDerived>
+bool MatrixBase<Derived>::isOrthogonal(const MatrixBase<OtherDerived>& other, const RealScalar& prec) const {
+  typename internal::nested_eval<Derived, 2>::type nested(derived());
+  typename internal::nested_eval<OtherDerived, 2>::type otherNested(other.derived());
+  return numext::abs2(nested.dot(otherNested)) <= prec * prec * nested.squaredNorm() * otherNested.squaredNorm();
+}
+
+/** \returns true if *this is approximately an unitary matrix,
+ *          within the precision given by \a prec. In the case where the \a Scalar
+ *          type is real numbers, a unitary matrix is an orthogonal matrix, whence the name.
+ *
+ * \note This can be used to check whether a family of vectors forms an orthonormal basis.
+ *       Indeed, \c m.isUnitary() returns true if and only if the columns (equivalently, the rows) of m form an
+ *       orthonormal basis.
+ *
+ * Example: \include MatrixBase_isUnitary.cpp
+ * Output: \verbinclude MatrixBase_isUnitary.out
+ */
+template <typename Derived>
+bool MatrixBase<Derived>::isUnitary(const RealScalar& prec) const {
+  typename internal::nested_eval<Derived, 1>::type self(derived());
+  for (Index i = 0; i < cols(); ++i) {
+    if (!internal::isApprox(self.col(i).squaredNorm(), static_cast<RealScalar>(1), prec)) return false;
+    for (Index j = 0; j < i; ++j)
+      if (!internal::isMuchSmallerThan(self.col(i).dot(self.col(j)), static_cast<Scalar>(1), prec)) return false;
+  }
+  return true;
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_DOT_H

include/Eigen/src/Core/EigenBase.h

@@ -0,0 +1,149 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_EIGENBASE_H
+#define EIGEN_EIGENBASE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class EigenBase
+ * \ingroup Core_Module
+ *
+ * Common base class for all classes T such that MatrixBase has an operator=(T) and a constructor MatrixBase(T).
+ *
+ * In other words, an EigenBase object is an object that can be copied into a MatrixBase.
+ *
+ * Besides MatrixBase-derived classes, this also includes special matrix classes such as diagonal matrices, etc.
+ *
+ * Notice that this class is trivial, it is only used to disambiguate overloaded functions.
+ *
+ * \sa \blank \ref TopicClassHierarchy
+ */
+template <typename Derived>
+struct EigenBase {
+  //   typedef typename internal::plain_matrix_type<Derived>::type PlainObject;
+
+  /** \brief The interface type of indices
+   * \details To change this, \c \#define the preprocessor symbol \c EIGEN_DEFAULT_DENSE_INDEX_TYPE.
+   * \sa StorageIndex, \ref TopicPreprocessorDirectives.
+   * DEPRECATED: Since Eigen 3.3, its usage is deprecated. Use Eigen::Index instead.
+   * Deprecation is not marked with a doxygen comment because there are too many existing usages to add the deprecation
+   * attribute.
+   */
+  typedef Eigen::Index Index;
+
+  // FIXME is it needed?
+  typedef typename internal::traits<Derived>::StorageKind StorageKind;
+
+  /** \returns a reference to the derived object */
+  EIGEN_DEVICE_FUNC constexpr Derived& derived() { return *static_cast<Derived*>(this); }
+  /** \returns a const reference to the derived object */
+  EIGEN_DEVICE_FUNC constexpr const Derived& derived() const { return *static_cast<const Derived*>(this); }
+
+  EIGEN_DEVICE_FUNC inline Derived& const_cast_derived() const {
+    return *static_cast<Derived*>(const_cast<EigenBase*>(this));
+  }
+  EIGEN_DEVICE_FUNC inline const Derived& const_derived() const { return *static_cast<const Derived*>(this); }
+
+  /** \returns the number of rows. \sa cols(), RowsAtCompileTime */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return derived().rows(); }
+  /** \returns the number of columns. \sa rows(), ColsAtCompileTime*/
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return derived().cols(); }
+  /** \returns the number of coefficients, which is rows()*cols().
+   * \sa rows(), cols(), SizeAtCompileTime. */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index size() const EIGEN_NOEXCEPT { return rows() * cols(); }
+
+  /** \internal Don't use it, but do the equivalent: \code dst = *this; \endcode */
+  template <typename Dest>
+  EIGEN_DEVICE_FUNC inline void evalTo(Dest& dst) const {
+    derived().evalTo(dst);
+  }
+
+  /** \internal Don't use it, but do the equivalent: \code dst += *this; \endcode */
+  template <typename Dest>
+  EIGEN_DEVICE_FUNC inline void addTo(Dest& dst) const {
+    // This is the default implementation,
+    // derived class can reimplement it in a more optimized way.
+    typename Dest::PlainObject res(rows(), cols());
+    evalTo(res);
+    dst += res;
+  }
+
+  /** \internal Don't use it, but do the equivalent: \code dst -= *this; \endcode */
+  template <typename Dest>
+  EIGEN_DEVICE_FUNC inline void subTo(Dest& dst) const {
+    // This is the default implementation,
+    // derived class can reimplement it in a more optimized way.
+    typename Dest::PlainObject res(rows(), cols());
+    evalTo(res);
+    dst -= res;
+  }
+
+  /** \internal Don't use it, but do the equivalent: \code dst.applyOnTheRight(*this); \endcode */
+  template <typename Dest>
+  EIGEN_DEVICE_FUNC inline void applyThisOnTheRight(Dest& dst) const {
+    // This is the default implementation,
+    // derived class can reimplement it in a more optimized way.
+    dst = dst * this->derived();
+  }
+
+  /** \internal Don't use it, but do the equivalent: \code dst.applyOnTheLeft(*this); \endcode */
+  template <typename Dest>
+  EIGEN_DEVICE_FUNC inline void applyThisOnTheLeft(Dest& dst) const {
+    // This is the default implementation,
+    // derived class can reimplement it in a more optimized way.
+    dst = this->derived() * dst;
+  }
+
+  template <typename Device>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DeviceWrapper<Derived, Device> device(Device& device);
+  template <typename Device>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DeviceWrapper<const Derived, Device> device(Device& device) const;
+};
+
+/***************************************************************************
+ * Implementation of matrix base methods
+ ***************************************************************************/
+
+/** \brief Copies the generic expression \a other into *this.
+ *
+ * \details The expression must provide a (templated) evalTo(Derived& dst) const
+ * function which does the actual job. In practice, this allows any user to write
+ * its own special matrix without having to modify MatrixBase
+ *
+ * \returns a reference to *this.
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC Derived& DenseBase<Derived>::operator=(const EigenBase<OtherDerived>& other) {
+  call_assignment(derived(), other.derived());
+  return derived();
+}
+
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC Derived& DenseBase<Derived>::operator+=(const EigenBase<OtherDerived>& other) {
+  call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar, typename OtherDerived::Scalar>());
+  return derived();
+}
+
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC Derived& DenseBase<Derived>::operator-=(const EigenBase<OtherDerived>& other) {
+  call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar, typename OtherDerived::Scalar>());
+  return derived();
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_EIGENBASE_H

include/Eigen/src/Core/Fill.h

@@ -0,0 +1,138 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2024 Charles Schlosser <cs.schlosser@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_FILL_H
+#define EIGEN_FILL_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Xpr>
+struct eigen_fill_helper : std::false_type {};
+
+template <typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols>
+struct eigen_fill_helper<Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols>> : std::true_type {};
+
+template <typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols>
+struct eigen_fill_helper<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols>> : std::true_type {};
+
+template <typename Xpr, int BlockRows, int BlockCols>
+struct eigen_fill_helper<Block<Xpr, BlockRows, BlockCols, /*InnerPanel*/ true>> : eigen_fill_helper<Xpr> {};
+
+template <typename Xpr, int BlockRows, int BlockCols>
+struct eigen_fill_helper<Block<Xpr, BlockRows, BlockCols, /*InnerPanel*/ false>>
+    : std::integral_constant<bool, eigen_fill_helper<Xpr>::value &&
+                                       (Xpr::IsRowMajor ? (BlockRows == 1) : (BlockCols == 1))> {};
+
+template <typename Xpr, int Options>
+struct eigen_fill_helper<Map<Xpr, Options, Stride<0, 0>>> : eigen_fill_helper<Xpr> {};
+
+template <typename Xpr, int Options, int OuterStride_>
+struct eigen_fill_helper<Map<Xpr, Options, Stride<OuterStride_, 0>>>
+    : std::integral_constant<bool, eigen_fill_helper<Xpr>::value &&
+                                       enum_eq_not_dynamic(OuterStride_, Xpr::InnerSizeAtCompileTime)> {};
+
+template <typename Xpr, int Options, int OuterStride_>
+struct eigen_fill_helper<Map<Xpr, Options, Stride<OuterStride_, 1>>>
+    : eigen_fill_helper<Map<Xpr, Options, Stride<OuterStride_, 0>>> {};
+
+template <typename Xpr, int Options, int InnerStride_>
+struct eigen_fill_helper<Map<Xpr, Options, InnerStride<InnerStride_>>>
+    : eigen_fill_helper<Map<Xpr, Options, Stride<0, InnerStride_>>> {};
+
+template <typename Xpr, int Options, int OuterStride_>
+struct eigen_fill_helper<Map<Xpr, Options, OuterStride<OuterStride_>>>
+    : eigen_fill_helper<Map<Xpr, Options, Stride<OuterStride_, 0>>> {};
+
+template <typename Xpr>
+struct eigen_fill_impl<Xpr, /*use_fill*/ false> {
+  using Scalar = typename Xpr::Scalar;
+  using Func = scalar_constant_op<Scalar>;
+  using PlainObject = typename Xpr::PlainObject;
+  using Constant = typename PlainObject::ConstantReturnType;
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const Scalar& val) {
+    const Constant src(dst.rows(), dst.cols(), val);
+    run(dst, src);
+  }
+  template <typename SrcXpr>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const SrcXpr& src) {
+    call_dense_assignment_loop(dst, src, assign_op<Scalar, Scalar>());
+  }
+};
+
+#if EIGEN_COMP_MSVC || defined(EIGEN_GPU_COMPILE_PHASE)
+template <typename Xpr>
+struct eigen_fill_impl<Xpr, /*use_fill*/ true> : eigen_fill_impl<Xpr, /*use_fill*/ false> {};
+#else
+template <typename Xpr>
+struct eigen_fill_impl<Xpr, /*use_fill*/ true> {
+  using Scalar = typename Xpr::Scalar;
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const Scalar& val) {
+    EIGEN_USING_STD(fill_n);
+    fill_n(dst.data(), dst.size(), val);
+  }
+  template <typename SrcXpr>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const SrcXpr& src) {
+    resize_if_allowed(dst, src, assign_op<Scalar, Scalar>());
+    const Scalar& val = src.functor()();
+    run(dst, val);
+  }
+};
+#endif
+
+template <typename Xpr>
+struct eigen_memset_helper {
+  static constexpr bool value = std::is_trivial<typename Xpr::Scalar>::value && eigen_fill_helper<Xpr>::value;
+};
+
+template <typename Xpr>
+struct eigen_zero_impl<Xpr, /*use_memset*/ false> {
+  using Scalar = typename Xpr::Scalar;
+  using PlainObject = typename Xpr::PlainObject;
+  using Zero = typename PlainObject::ZeroReturnType;
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst) {
+    const Zero src(dst.rows(), dst.cols());
+    run(dst, src);
+  }
+  template <typename SrcXpr>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const SrcXpr& src) {
+    call_dense_assignment_loop(dst, src, assign_op<Scalar, Scalar>());
+  }
+};
+
+template <typename Xpr>
+struct eigen_zero_impl<Xpr, /*use_memset*/ true> {
+  using Scalar = typename Xpr::Scalar;
+  static constexpr size_t max_bytes = (std::numeric_limits<std::ptrdiff_t>::max)();
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst) {
+    const size_t num_bytes = dst.size() * sizeof(Scalar);
+    if (num_bytes == 0) return;
+    void* dst_ptr = static_cast<void*>(dst.data());
+#ifndef EIGEN_NO_DEBUG
+    if (num_bytes > max_bytes) throw_std_bad_alloc();
+    eigen_assert((dst_ptr != nullptr) && "null pointer dereference error!");
+#endif
+    EIGEN_USING_STD(memset);
+    memset(dst_ptr, 0, num_bytes);
+  }
+  template <typename SrcXpr>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const SrcXpr& src) {
+    resize_if_allowed(dst, src, assign_op<Scalar, Scalar>());
+    run(dst);
+  }
+};
+
+}  // namespace internal
+}  // namespace Eigen
+
+#endif  // EIGEN_FILL_H

include/Eigen/src/Core/ForceAlignedAccess.h

@@ -0,0 +1,131 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_FORCEALIGNEDACCESS_H
+#define EIGEN_FORCEALIGNEDACCESS_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class ForceAlignedAccess
+ * \ingroup Core_Module
+ *
+ * \brief Enforce aligned packet loads and stores regardless of what is requested
+ *
+ * \param ExpressionType the type of the object of which we are forcing aligned packet access
+ *
+ * This class is the return type of MatrixBase::forceAlignedAccess()
+ * and most of the time this is the only way it is used.
+ *
+ * \sa MatrixBase::forceAlignedAccess()
+ */
+
+namespace internal {
+template <typename ExpressionType>
+struct traits<ForceAlignedAccess<ExpressionType>> : public traits<ExpressionType> {};
+}  // namespace internal
+
+template <typename ExpressionType>
+class ForceAlignedAccess : public internal::dense_xpr_base<ForceAlignedAccess<ExpressionType>>::type {
+ public:
+  typedef typename internal::dense_xpr_base<ForceAlignedAccess>::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(ForceAlignedAccess)
+
+  EIGEN_DEVICE_FUNC explicit inline ForceAlignedAccess(const ExpressionType& matrix) : m_expression(matrix) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_expression.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_expression.cols(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT {
+    return m_expression.outerStride();
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT {
+    return m_expression.innerStride();
+  }
+
+  EIGEN_DEVICE_FUNC inline const CoeffReturnType coeff(Index row, Index col) const {
+    return m_expression.coeff(row, col);
+  }
+
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index row, Index col) {
+    return m_expression.const_cast_derived().coeffRef(row, col);
+  }
+
+  EIGEN_DEVICE_FUNC inline const CoeffReturnType coeff(Index index) const { return m_expression.coeff(index); }
+
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index index) { return m_expression.const_cast_derived().coeffRef(index); }
+
+  template <int LoadMode>
+  inline const PacketScalar packet(Index row, Index col) const {
+    return m_expression.template packet<Aligned>(row, col);
+  }
+
+  template <int LoadMode>
+  inline void writePacket(Index row, Index col, const PacketScalar& x) {
+    m_expression.const_cast_derived().template writePacket<Aligned>(row, col, x);
+  }
+
+  template <int LoadMode>
+  inline const PacketScalar packet(Index index) const {
+    return m_expression.template packet<Aligned>(index);
+  }
+
+  template <int LoadMode>
+  inline void writePacket(Index index, const PacketScalar& x) {
+    m_expression.const_cast_derived().template writePacket<Aligned>(index, x);
+  }
+
+  EIGEN_DEVICE_FUNC operator const ExpressionType&() const { return m_expression; }
+
+ protected:
+  const ExpressionType& m_expression;
+
+ private:
+  ForceAlignedAccess& operator=(const ForceAlignedAccess&);
+};
+
+/** \returns an expression of *this with forced aligned access
+ * \sa forceAlignedAccessIf(),class ForceAlignedAccess
+ */
+template <typename Derived>
+inline const ForceAlignedAccess<Derived> MatrixBase<Derived>::forceAlignedAccess() const {
+  return ForceAlignedAccess<Derived>(derived());
+}
+
+/** \returns an expression of *this with forced aligned access
+ * \sa forceAlignedAccessIf(), class ForceAlignedAccess
+ */
+template <typename Derived>
+inline ForceAlignedAccess<Derived> MatrixBase<Derived>::forceAlignedAccess() {
+  return ForceAlignedAccess<Derived>(derived());
+}
+
+/** \returns an expression of *this with forced aligned access if \a Enable is true.
+ * \sa forceAlignedAccess(), class ForceAlignedAccess
+ */
+template <typename Derived>
+template <bool Enable>
+inline add_const_on_value_type_t<std::conditional_t<Enable, ForceAlignedAccess<Derived>, Derived&>>
+MatrixBase<Derived>::forceAlignedAccessIf() const {
+  return derived();  // FIXME This should not work but apparently is never used
+}
+
+/** \returns an expression of *this with forced aligned access if \a Enable is true.
+ * \sa forceAlignedAccess(), class ForceAlignedAccess
+ */
+template <typename Derived>
+template <bool Enable>
+inline std::conditional_t<Enable, ForceAlignedAccess<Derived>, Derived&> MatrixBase<Derived>::forceAlignedAccessIf() {
+  return derived();  // FIXME This should not work but apparently is never used
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_FORCEALIGNEDACCESS_H

include/Eigen/src/Core/Fuzzy.h

@@ -0,0 +1,132 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_FUZZY_H
+#define EIGEN_FUZZY_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Derived, typename OtherDerived, bool is_integer = NumTraits<typename Derived::Scalar>::IsInteger>
+struct isApprox_selector {
+  EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar& prec) {
+    typename internal::nested_eval<Derived, 2>::type nested(x);
+    typename internal::nested_eval<OtherDerived, 2>::type otherNested(y);
+    return (nested.matrix() - otherNested.matrix()).cwiseAbs2().sum() <=
+           prec * prec * numext::mini(nested.cwiseAbs2().sum(), otherNested.cwiseAbs2().sum());
+  }
+};
+
+template <typename Derived, typename OtherDerived>
+struct isApprox_selector<Derived, OtherDerived, true> {
+  EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar&) {
+    return x.matrix() == y.matrix();
+  }
+};
+
+template <typename Derived, typename OtherDerived, bool is_integer = NumTraits<typename Derived::Scalar>::IsInteger>
+struct isMuchSmallerThan_object_selector {
+  EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar& prec) {
+    return x.cwiseAbs2().sum() <= numext::abs2(prec) * y.cwiseAbs2().sum();
+  }
+};
+
+template <typename Derived, typename OtherDerived>
+struct isMuchSmallerThan_object_selector<Derived, OtherDerived, true> {
+  EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived&, const typename Derived::RealScalar&) {
+    return x.matrix() == Derived::Zero(x.rows(), x.cols()).matrix();
+  }
+};
+
+template <typename Derived, bool is_integer = NumTraits<typename Derived::Scalar>::IsInteger>
+struct isMuchSmallerThan_scalar_selector {
+  EIGEN_DEVICE_FUNC static bool run(const Derived& x, const typename Derived::RealScalar& y,
+                                    const typename Derived::RealScalar& prec) {
+    return x.cwiseAbs2().sum() <= numext::abs2(prec * y);
+  }
+};
+
+template <typename Derived>
+struct isMuchSmallerThan_scalar_selector<Derived, true> {
+  EIGEN_DEVICE_FUNC static bool run(const Derived& x, const typename Derived::RealScalar&,
+                                    const typename Derived::RealScalar&) {
+    return x.matrix() == Derived::Zero(x.rows(), x.cols()).matrix();
+  }
+};
+
+}  // end namespace internal
+
+/** \returns \c true if \c *this is approximately equal to \a other, within the precision
+ * determined by \a prec.
+ *
+ * \note The fuzzy compares are done multiplicatively. Two vectors \f$ v \f$ and \f$ w \f$
+ * are considered to be approximately equal within precision \f$ p \f$ if
+ * \f[ \Vert v - w \Vert \leqslant p\,\min(\Vert v\Vert, \Vert w\Vert). \f]
+ * For matrices, the comparison is done using the Hilbert-Schmidt norm (aka Frobenius norm
+ * L2 norm).
+ *
+ * \note Because of the multiplicativeness of this comparison, one can't use this function
+ * to check whether \c *this is approximately equal to the zero matrix or vector.
+ * Indeed, \c isApprox(zero) returns false unless \c *this itself is exactly the zero matrix
+ * or vector. If you want to test whether \c *this is zero, use internal::isMuchSmallerThan(const
+ * RealScalar&, RealScalar) instead.
+ *
+ * \sa internal::isMuchSmallerThan(const RealScalar&, RealScalar) const
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isApprox(const DenseBase<OtherDerived>& other,
+                                                    const RealScalar& prec) const {
+  return internal::isApprox_selector<Derived, OtherDerived>::run(derived(), other.derived(), prec);
+}
+
+/** \returns \c true if the norm of \c *this is much smaller than \a other,
+ * within the precision determined by \a prec.
+ *
+ * \note The fuzzy compares are done multiplicatively. A vector \f$ v \f$ is
+ * considered to be much smaller than \f$ x \f$ within precision \f$ p \f$ if
+ * \f[ \Vert v \Vert \leqslant p\,\vert x\vert. \f]
+ *
+ * For matrices, the comparison is done using the Hilbert-Schmidt norm. For this reason,
+ * the value of the reference scalar \a other should come from the Hilbert-Schmidt norm
+ * of a reference matrix of same dimensions.
+ *
+ * \sa isApprox(), isMuchSmallerThan(const DenseBase<OtherDerived>&, RealScalar) const
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isMuchSmallerThan(const typename NumTraits<Scalar>::Real& other,
+                                                             const RealScalar& prec) const {
+  return internal::isMuchSmallerThan_scalar_selector<Derived>::run(derived(), other, prec);
+}
+
+/** \returns \c true if the norm of \c *this is much smaller than the norm of \a other,
+ * within the precision determined by \a prec.
+ *
+ * \note The fuzzy compares are done multiplicatively. A vector \f$ v \f$ is
+ * considered to be much smaller than a vector \f$ w \f$ within precision \f$ p \f$ if
+ * \f[ \Vert v \Vert \leqslant p\,\Vert w\Vert. \f]
+ * For matrices, the comparison is done using the Hilbert-Schmidt norm.
+ *
+ * \sa isApprox(), isMuchSmallerThan(const RealScalar&, RealScalar) const
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isMuchSmallerThan(const DenseBase<OtherDerived>& other,
+                                                             const RealScalar& prec) const {
+  return internal::isMuchSmallerThan_object_selector<Derived, OtherDerived>::run(derived(), other.derived(), prec);
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_FUZZY_H

include/Eigen/src/Core/GeneralProduct.h

@@ -0,0 +1,519 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2008-2011 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_GENERAL_PRODUCT_H
+#define EIGEN_GENERAL_PRODUCT_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+enum { Large = 2, Small = 3 };
+
+// Define the threshold value to fallback from the generic matrix-matrix product
+// implementation (heavy) to the lightweight coeff-based product one.
+// See generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemmProduct>
+// in products/GeneralMatrixMatrix.h for more details.
+// TODO This threshold should also be used in the compile-time selector below.
+#ifndef EIGEN_GEMM_TO_COEFFBASED_THRESHOLD
+// This default value has been obtained on a Haswell architecture.
+#define EIGEN_GEMM_TO_COEFFBASED_THRESHOLD 20
+#endif
+
+namespace internal {
+
+template <int Rows, int Cols, int Depth>
+struct product_type_selector;
+
+template <int Size, int MaxSize>
+struct product_size_category {
+  enum {
+#ifndef EIGEN_GPU_COMPILE_PHASE
+    is_large = MaxSize == Dynamic || Size >= EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD ||
+               (Size == Dynamic && MaxSize >= EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD),
+#else
+    is_large = 0,
+#endif
+    value = is_large    ? Large
+            : Size == 1 ? 1
+                        : Small
+  };
+};
+
+template <typename Lhs, typename Rhs>
+struct product_type {
+  typedef remove_all_t<Lhs> Lhs_;
+  typedef remove_all_t<Rhs> Rhs_;
+  enum {
+    MaxRows = traits<Lhs_>::MaxRowsAtCompileTime,
+    Rows = traits<Lhs_>::RowsAtCompileTime,
+    MaxCols = traits<Rhs_>::MaxColsAtCompileTime,
+    Cols = traits<Rhs_>::ColsAtCompileTime,
+    MaxDepth = min_size_prefer_fixed(traits<Lhs_>::MaxColsAtCompileTime, traits<Rhs_>::MaxRowsAtCompileTime),
+    Depth = min_size_prefer_fixed(traits<Lhs_>::ColsAtCompileTime, traits<Rhs_>::RowsAtCompileTime)
+  };
+
+  // the splitting into different lines of code here, introducing the _select enums and the typedef below,
+  // is to work around an internal compiler error with gcc 4.1 and 4.2.
+ private:
+  enum {
+    rows_select = product_size_category<Rows, MaxRows>::value,
+    cols_select = product_size_category<Cols, MaxCols>::value,
+    depth_select = product_size_category<Depth, MaxDepth>::value
+  };
+  typedef product_type_selector<rows_select, cols_select, depth_select> selector;
+
+ public:
+  enum { value = selector::ret, ret = selector::ret };
+#ifdef EIGEN_DEBUG_PRODUCT
+  static void debug() {
+    EIGEN_DEBUG_VAR(Rows);
+    EIGEN_DEBUG_VAR(Cols);
+    EIGEN_DEBUG_VAR(Depth);
+    EIGEN_DEBUG_VAR(rows_select);
+    EIGEN_DEBUG_VAR(cols_select);
+    EIGEN_DEBUG_VAR(depth_select);
+    EIGEN_DEBUG_VAR(value);
+  }
+#endif
+};
+
+/* The following allows to select the kind of product at compile time
+ * based on the three dimensions of the product.
+ * This is a compile time mapping from {1,Small,Large}^3 -> {product types} */
+// FIXME I'm not sure the current mapping is the ideal one.
+template <int M, int N>
+struct product_type_selector<M, N, 1> {
+  enum { ret = OuterProduct };
+};
+template <int M>
+struct product_type_selector<M, 1, 1> {
+  enum { ret = LazyCoeffBasedProductMode };
+};
+template <int N>
+struct product_type_selector<1, N, 1> {
+  enum { ret = LazyCoeffBasedProductMode };
+};
+template <int Depth>
+struct product_type_selector<1, 1, Depth> {
+  enum { ret = InnerProduct };
+};
+template <>
+struct product_type_selector<1, 1, 1> {
+  enum { ret = InnerProduct };
+};
+template <>
+struct product_type_selector<Small, 1, Small> {
+  enum { ret = CoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<1, Small, Small> {
+  enum { ret = CoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<Small, Small, Small> {
+  enum { ret = CoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<Small, Small, 1> {
+  enum { ret = LazyCoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<Small, Large, 1> {
+  enum { ret = LazyCoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<Large, Small, 1> {
+  enum { ret = LazyCoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<1, Large, Small> {
+  enum { ret = CoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<1, Large, Large> {
+  enum { ret = GemvProduct };
+};
+template <>
+struct product_type_selector<1, Small, Large> {
+  enum { ret = CoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<Large, 1, Small> {
+  enum { ret = CoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<Large, 1, Large> {
+  enum { ret = GemvProduct };
+};
+template <>
+struct product_type_selector<Small, 1, Large> {
+  enum { ret = CoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<Small, Small, Large> {
+  enum { ret = GemmProduct };
+};
+template <>
+struct product_type_selector<Large, Small, Large> {
+  enum { ret = GemmProduct };
+};
+template <>
+struct product_type_selector<Small, Large, Large> {
+  enum { ret = GemmProduct };
+};
+template <>
+struct product_type_selector<Large, Large, Large> {
+  enum { ret = GemmProduct };
+};
+template <>
+struct product_type_selector<Large, Small, Small> {
+  enum { ret = CoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<Small, Large, Small> {
+  enum { ret = CoeffBasedProductMode };
+};
+template <>
+struct product_type_selector<Large, Large, Small> {
+  enum { ret = GemmProduct };
+};
+
+}  // end namespace internal
+
+/***********************************************************************
+ *  Implementation of Inner Vector Vector Product
+ ***********************************************************************/
+
+// FIXME : maybe the "inner product" could return a Scalar
+// instead of a 1x1 matrix ??
+// Pro: more natural for the user
+// Cons: this could be a problem if in a meta unrolled algorithm a matrix-matrix
+// product ends up to a row-vector times col-vector product... To tackle this use
+// case, we could have a specialization for Block<MatrixType,1,1> with: operator=(Scalar x);
+
+/***********************************************************************
+ *  Implementation of Outer Vector Vector Product
+ ***********************************************************************/
+
+/***********************************************************************
+ *  Implementation of General Matrix Vector Product
+ ***********************************************************************/
+
+/*  According to the shape/flags of the matrix we have to distinghish 3 different cases:
+ *   1 - the matrix is col-major, BLAS compatible and M is large => call fast BLAS-like colmajor routine
+ *   2 - the matrix is row-major, BLAS compatible and N is large => call fast BLAS-like rowmajor routine
+ *   3 - all other cases are handled using a simple loop along the outer-storage direction.
+ *  Therefore we need a lower level meta selector.
+ *  Furthermore, if the matrix is the rhs, then the product has to be transposed.
+ */
+namespace internal {
+
+template <int Side, int StorageOrder, bool BlasCompatible>
+struct gemv_dense_selector;
+
+}  // end namespace internal
+
+namespace internal {
+
+template <typename Scalar, int Size, int MaxSize, bool Cond>
+struct gemv_static_vector_if;
+
+template <typename Scalar, int Size, int MaxSize>
+struct gemv_static_vector_if<Scalar, Size, MaxSize, false> {
+  EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC constexpr Scalar* data() {
+    eigen_internal_assert(false && "should never be called");
+    return 0;
+  }
+};
+
+template <typename Scalar, int Size>
+struct gemv_static_vector_if<Scalar, Size, Dynamic, true> {
+  EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC constexpr Scalar* data() { return 0; }
+};
+
+template <typename Scalar, int Size, int MaxSize>
+struct gemv_static_vector_if<Scalar, Size, MaxSize, true> {
+#if EIGEN_MAX_STATIC_ALIGN_BYTES != 0
+  internal::plain_array<Scalar, internal::min_size_prefer_fixed(Size, MaxSize), 0, AlignedMax> m_data;
+  EIGEN_STRONG_INLINE constexpr Scalar* data() { return m_data.array; }
+#else
+  // Some architectures cannot align on the stack,
+  // => let's manually enforce alignment by allocating more data and return the address of the first aligned element.
+  internal::plain_array<Scalar, internal::min_size_prefer_fixed(Size, MaxSize) + EIGEN_MAX_ALIGN_BYTES, 0> m_data;
+  EIGEN_STRONG_INLINE constexpr Scalar* data() {
+    return reinterpret_cast<Scalar*>((std::uintptr_t(m_data.array) & ~(std::size_t(EIGEN_MAX_ALIGN_BYTES - 1))) +
+                                     EIGEN_MAX_ALIGN_BYTES);
+  }
+#endif
+};
+
+// The vector is on the left => transposition
+template <int StorageOrder, bool BlasCompatible>
+struct gemv_dense_selector<OnTheLeft, StorageOrder, BlasCompatible> {
+  template <typename Lhs, typename Rhs, typename Dest>
+  static void run(const Lhs& lhs, const Rhs& rhs, Dest& dest, const typename Dest::Scalar& alpha) {
+    Transpose<Dest> destT(dest);
+    enum { OtherStorageOrder = StorageOrder == RowMajor ? ColMajor : RowMajor };
+    gemv_dense_selector<OnTheRight, OtherStorageOrder, BlasCompatible>::run(rhs.transpose(), lhs.transpose(), destT,
+                                                                            alpha);
+  }
+};
+
+template <>
+struct gemv_dense_selector<OnTheRight, ColMajor, true> {
+  template <typename Lhs, typename Rhs, typename Dest>
+  static inline void run(const Lhs& lhs, const Rhs& rhs, Dest& dest, const typename Dest::Scalar& alpha) {
+    typedef typename Lhs::Scalar LhsScalar;
+    typedef typename Rhs::Scalar RhsScalar;
+    typedef typename Dest::Scalar ResScalar;
+
+    typedef internal::blas_traits<Lhs> LhsBlasTraits;
+    typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType;
+    typedef internal::blas_traits<Rhs> RhsBlasTraits;
+    typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType;
+
+    typedef Map<Matrix<ResScalar, Dynamic, 1>, plain_enum_min(AlignedMax, internal::packet_traits<ResScalar>::size)>
+        MappedDest;
+
+    ActualLhsType actualLhs = LhsBlasTraits::extract(lhs);
+    ActualRhsType actualRhs = RhsBlasTraits::extract(rhs);
+
+    ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs);
+
+    // make sure Dest is a compile-time vector type (bug 1166)
+    typedef std::conditional_t<Dest::IsVectorAtCompileTime, Dest, typename Dest::ColXpr> ActualDest;
+
+    enum {
+      // FIXME find a way to allow an inner stride on the result if packet_traits<Scalar>::size==1
+      // on, the other hand it is good for the cache to pack the vector anyways...
+      EvalToDestAtCompileTime = (ActualDest::InnerStrideAtCompileTime == 1),
+      ComplexByReal = (NumTraits<LhsScalar>::IsComplex) && (!NumTraits<RhsScalar>::IsComplex),
+      MightCannotUseDest = ((!EvalToDestAtCompileTime) || ComplexByReal) && (ActualDest::MaxSizeAtCompileTime != 0)
+    };
+
+    typedef const_blas_data_mapper<LhsScalar, Index, ColMajor> LhsMapper;
+    typedef const_blas_data_mapper<RhsScalar, Index, RowMajor> RhsMapper;
+    RhsScalar compatibleAlpha = get_factor<ResScalar, RhsScalar>::run(actualAlpha);
+
+    if (!MightCannotUseDest) {
+      // shortcut if we are sure to be able to use dest directly,
+      // this ease the compiler to generate cleaner and more optimzized code for most common cases
+      general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, LhsBlasTraits::NeedToConjugate, RhsScalar,
+                                    RhsMapper, RhsBlasTraits::NeedToConjugate>::run(actualLhs.rows(), actualLhs.cols(),
+                                                                                    LhsMapper(actualLhs.data(),
+                                                                                              actualLhs.outerStride()),
+                                                                                    RhsMapper(actualRhs.data(),
+                                                                                              actualRhs.innerStride()),
+                                                                                    dest.data(), 1, compatibleAlpha);
+    } else {
+      gemv_static_vector_if<ResScalar, ActualDest::SizeAtCompileTime, ActualDest::MaxSizeAtCompileTime,
+                            MightCannotUseDest>
+          static_dest;
+
+      const bool alphaIsCompatible = (!ComplexByReal) || (numext::is_exactly_zero(numext::imag(actualAlpha)));
+      const bool evalToDest = EvalToDestAtCompileTime && alphaIsCompatible;
+
+      ei_declare_aligned_stack_constructed_variable(ResScalar, actualDestPtr, dest.size(),
+                                                    evalToDest ? dest.data() : static_dest.data());
+
+      if (!evalToDest) {
+#ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN
+        constexpr int Size = Dest::SizeAtCompileTime;
+        Index size = dest.size();
+        EIGEN_DENSE_STORAGE_CTOR_PLUGIN
+#endif
+        if (!alphaIsCompatible) {
+          MappedDest(actualDestPtr, dest.size()).setZero();
+          compatibleAlpha = RhsScalar(1);
+        } else
+          MappedDest(actualDestPtr, dest.size()) = dest;
+      }
+
+      general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, LhsBlasTraits::NeedToConjugate, RhsScalar,
+                                    RhsMapper, RhsBlasTraits::NeedToConjugate>::run(actualLhs.rows(), actualLhs.cols(),
+                                                                                    LhsMapper(actualLhs.data(),
+                                                                                              actualLhs.outerStride()),
+                                                                                    RhsMapper(actualRhs.data(),
+                                                                                              actualRhs.innerStride()),
+                                                                                    actualDestPtr, 1, compatibleAlpha);
+
+      if (!evalToDest) {
+        if (!alphaIsCompatible)
+          dest.matrix() += actualAlpha * MappedDest(actualDestPtr, dest.size());
+        else
+          dest = MappedDest(actualDestPtr, dest.size());
+      }
+    }
+  }
+};
+
+template <>
+struct gemv_dense_selector<OnTheRight, RowMajor, true> {
+  template <typename Lhs, typename Rhs, typename Dest>
+  static void run(const Lhs& lhs, const Rhs& rhs, Dest& dest, const typename Dest::Scalar& alpha) {
+    typedef typename Lhs::Scalar LhsScalar;
+    typedef typename Rhs::Scalar RhsScalar;
+    typedef typename Dest::Scalar ResScalar;
+
+    typedef internal::blas_traits<Lhs> LhsBlasTraits;
+    typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType;
+    typedef internal::blas_traits<Rhs> RhsBlasTraits;
+    typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType;
+    typedef internal::remove_all_t<ActualRhsType> ActualRhsTypeCleaned;
+
+    std::add_const_t<ActualLhsType> actualLhs = LhsBlasTraits::extract(lhs);
+    std::add_const_t<ActualRhsType> actualRhs = RhsBlasTraits::extract(rhs);
+
+    ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs);
+
+    enum {
+      // FIXME find a way to allow an inner stride on the result if packet_traits<Scalar>::size==1
+      // on, the other hand it is good for the cache to pack the vector anyways...
+      DirectlyUseRhs =
+          ActualRhsTypeCleaned::InnerStrideAtCompileTime == 1 || ActualRhsTypeCleaned::MaxSizeAtCompileTime == 0
+    };
+
+    gemv_static_vector_if<RhsScalar, ActualRhsTypeCleaned::SizeAtCompileTime,
+                          ActualRhsTypeCleaned::MaxSizeAtCompileTime, !DirectlyUseRhs>
+        static_rhs;
+
+    ei_declare_aligned_stack_constructed_variable(
+        RhsScalar, actualRhsPtr, actualRhs.size(),
+        DirectlyUseRhs ? const_cast<RhsScalar*>(actualRhs.data()) : static_rhs.data());
+
+    if (!DirectlyUseRhs) {
+#ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN
+      constexpr int Size = ActualRhsTypeCleaned::SizeAtCompileTime;
+      Index size = actualRhs.size();
+      EIGEN_DENSE_STORAGE_CTOR_PLUGIN
+#endif
+      Map<typename ActualRhsTypeCleaned::PlainObject>(actualRhsPtr, actualRhs.size()) = actualRhs;
+    }
+
+    typedef const_blas_data_mapper<LhsScalar, Index, RowMajor> LhsMapper;
+    typedef const_blas_data_mapper<RhsScalar, Index, ColMajor> RhsMapper;
+    general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, LhsBlasTraits::NeedToConjugate, RhsScalar,
+                                  RhsMapper, RhsBlasTraits::NeedToConjugate>::
+        run(actualLhs.rows(), actualLhs.cols(), LhsMapper(actualLhs.data(), actualLhs.outerStride()),
+            RhsMapper(actualRhsPtr, 1), dest.data(),
+            dest.col(0).innerStride(),  // NOTE  if dest is not a vector at compile-time, then dest.innerStride() might
+                                        // be wrong. (bug 1166)
+            actualAlpha);
+  }
+};
+
+template <>
+struct gemv_dense_selector<OnTheRight, ColMajor, false> {
+  template <typename Lhs, typename Rhs, typename Dest>
+  static void run(const Lhs& lhs, const Rhs& rhs, Dest& dest, const typename Dest::Scalar& alpha) {
+    EIGEN_STATIC_ASSERT((!nested_eval<Lhs, 1>::Evaluate),
+                        EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE);
+    // TODO if rhs is large enough it might be beneficial to make sure that dest is sequentially stored in memory,
+    // otherwise use a temp
+    typename nested_eval<Rhs, 1>::type actual_rhs(rhs);
+    const Index size = rhs.rows();
+    for (Index k = 0; k < size; ++k) dest += (alpha * actual_rhs.coeff(k)) * lhs.col(k);
+  }
+};
+
+template <>
+struct gemv_dense_selector<OnTheRight, RowMajor, false> {
+  template <typename Lhs, typename Rhs, typename Dest>
+  static void run(const Lhs& lhs, const Rhs& rhs, Dest& dest, const typename Dest::Scalar& alpha) {
+    EIGEN_STATIC_ASSERT((!nested_eval<Lhs, 1>::Evaluate),
+                        EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE);
+    typename nested_eval<Rhs, Lhs::RowsAtCompileTime>::type actual_rhs(rhs);
+    const Index rows = dest.rows();
+    for (Index i = 0; i < rows; ++i)
+      dest.coeffRef(i) += alpha * (lhs.row(i).cwiseProduct(actual_rhs.transpose())).sum();
+  }
+};
+
+}  // end namespace internal
+
+/***************************************************************************
+ * Implementation of matrix base methods
+ ***************************************************************************/
+
+/** \returns the matrix product of \c *this and \a other.
+ *
+ * \note If instead of the matrix product you want the coefficient-wise product, see Cwise::operator*().
+ *
+ * \sa lazyProduct(), operator*=(const MatrixBase&), Cwise::operator*()
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Product<Derived, OtherDerived> MatrixBase<Derived>::operator*(
+    const MatrixBase<OtherDerived>& other) const {
+  // A note regarding the function declaration: In MSVC, this function will sometimes
+  // not be inlined since DenseStorage is an unwindable object for dynamic
+  // matrices and product types are holding a member to store the result.
+  // Thus it does not help tagging this function with EIGEN_STRONG_INLINE.
+  enum {
+    ProductIsValid = Derived::ColsAtCompileTime == Dynamic || OtherDerived::RowsAtCompileTime == Dynamic ||
+                     int(Derived::ColsAtCompileTime) == int(OtherDerived::RowsAtCompileTime),
+    AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime,
+    SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived, OtherDerived)
+  };
+  // note to the lost user:
+  //    * for a dot product use: v1.dot(v2)
+  //    * for a coeff-wise product use: v1.cwiseProduct(v2)
+  EIGEN_STATIC_ASSERT(
+      ProductIsValid || !(AreVectors && SameSizes),
+      INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS)
+  EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors),
+                      INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION)
+  EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT)
+#ifdef EIGEN_DEBUG_PRODUCT
+  internal::product_type<Derived, OtherDerived>::debug();
+#endif
+
+  return Product<Derived, OtherDerived>(derived(), other.derived());
+}
+
+/** \returns an expression of the matrix product of \c *this and \a other without implicit evaluation.
+ *
+ * The returned product will behave like any other expressions: the coefficients of the product will be
+ * computed once at a time as requested. This might be useful in some extremely rare cases when only
+ * a small and no coherent fraction of the result's coefficients have to be computed.
+ *
+ * \warning This version of the matrix product can be much much slower. So use it only if you know
+ * what you are doing and that you measured a true speed improvement.
+ *
+ * \sa operator*(const MatrixBase&)
+ */
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Product<Derived, OtherDerived, LazyProduct>
+MatrixBase<Derived>::lazyProduct(const MatrixBase<OtherDerived>& other) const {
+  enum {
+    ProductIsValid = Derived::ColsAtCompileTime == Dynamic || OtherDerived::RowsAtCompileTime == Dynamic ||
+                     int(Derived::ColsAtCompileTime) == int(OtherDerived::RowsAtCompileTime),
+    AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime,
+    SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived, OtherDerived)
+  };
+  // note to the lost user:
+  //    * for a dot product use: v1.dot(v2)
+  //    * for a coeff-wise product use: v1.cwiseProduct(v2)
+  EIGEN_STATIC_ASSERT(
+      ProductIsValid || !(AreVectors && SameSizes),
+      INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS)
+  EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors),
+                      INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION)
+  EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT)
+
+  return Product<Derived, OtherDerived, LazyProduct>(derived(), other.derived());
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_PRODUCT_H

include/Eigen/src/Core/GlobalFunctions.h

@@ -0,0 +1,230 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2010-2016 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_GLOBAL_FUNCTIONS_H
+#define EIGEN_GLOBAL_FUNCTIONS_H
+
+#ifdef EIGEN_PARSED_BY_DOXYGEN
+
+#define EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(NAME, FUNCTOR, DOC_OP, DOC_DETAILS)                                    \
+  /** \returns an expression of the coefficient-wise DOC_OP of \a x                                             \
+                                                                                                              \ \
+    DOC_DETAILS                                                                                                 \
+                                                                                                              \ \
+    \sa <a href="group__CoeffwiseMathFunctions.html#cwisetable_##NAME">Math functions</a>, class CwiseUnaryOp   \
+    */                                                                                                          \
+  template <typename Derived>                                                                                   \
+  inline const Eigen::CwiseUnaryOp<Eigen::internal::FUNCTOR<typename Derived::Scalar>, const Derived> NAME(     \
+      const Eigen::ArrayBase<Derived>& x);
+
+#else
+
+#define EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(NAME, FUNCTOR, DOC_OP, DOC_DETAILS)                                    \
+  template <typename Derived>                                                                                   \
+  inline const Eigen::CwiseUnaryOp<Eigen::internal::FUNCTOR<typename Derived::Scalar>, const Derived>(NAME)(    \
+      const Eigen::ArrayBase<Derived>& x) {                                                                     \
+    return Eigen::CwiseUnaryOp<Eigen::internal::FUNCTOR<typename Derived::Scalar>, const Derived>(x.derived()); \
+  }
+
+#endif  // EIGEN_PARSED_BY_DOXYGEN
+
+#define EIGEN_ARRAY_DECLARE_GLOBAL_EIGEN_UNARY(NAME, FUNCTOR)                                                  \
+                                                                                                               \
+  template <typename Derived>                                                                                  \
+  struct NAME##_retval<ArrayBase<Derived> > {                                                                  \
+    typedef const Eigen::CwiseUnaryOp<Eigen::internal::FUNCTOR<typename Derived::Scalar>, const Derived> type; \
+  };                                                                                                           \
+  template <typename Derived>                                                                                  \
+  struct NAME##_impl<ArrayBase<Derived> > {                                                                    \
+    static inline typename NAME##_retval<ArrayBase<Derived> >::type run(const Eigen::ArrayBase<Derived>& x) {  \
+      return typename NAME##_retval<ArrayBase<Derived> >::type(x.derived());                                   \
+    }                                                                                                          \
+  };
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(real, scalar_real_op, real part,\sa ArrayBase::real)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(imag, scalar_imag_op, imaginary part,\sa ArrayBase::imag)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(conj, scalar_conjugate_op, complex conjugate,\sa ArrayBase::conjugate)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(inverse, scalar_inverse_op, inverse,\sa ArrayBase::inverse)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sin, scalar_sin_op, sine,\sa ArrayBase::sin)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cos, scalar_cos_op, cosine,\sa ArrayBase::cos)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(tan, scalar_tan_op, tangent,\sa ArrayBase::tan)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(atan, scalar_atan_op, arc - tangent,\sa ArrayBase::atan)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(asin, scalar_asin_op, arc - sine,\sa ArrayBase::asin)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(acos, scalar_acos_op, arc - consine,\sa ArrayBase::acos)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sinh, scalar_sinh_op, hyperbolic sine,\sa ArrayBase::sinh)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cosh, scalar_cosh_op, hyperbolic cosine,\sa ArrayBase::cosh)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(tanh, scalar_tanh_op, hyperbolic tangent,\sa ArrayBase::tanh)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(asinh, scalar_asinh_op, inverse hyperbolic sine,\sa ArrayBase::asinh)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(acosh, scalar_acosh_op, inverse hyperbolic cosine,\sa ArrayBase::acosh)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(atanh, scalar_atanh_op, inverse hyperbolic tangent,\sa ArrayBase::atanh)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(logistic, scalar_logistic_op, logistic function,\sa ArrayBase::logistic)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(lgamma, scalar_lgamma_op,
+                                 natural logarithm of the gamma function,\sa ArrayBase::lgamma)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(digamma, scalar_digamma_op, derivative of lgamma,\sa ArrayBase::digamma)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(erf, scalar_erf_op, error function,\sa ArrayBase::erf)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(erfc, scalar_erfc_op, complement error function,\sa ArrayBase::erfc)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(ndtri, scalar_ndtri_op, inverse normal distribution function,\sa ArrayBase::ndtri)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(exp, scalar_exp_op, exponential,\sa ArrayBase::exp)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(exp2, scalar_exp2_op, exponential,\sa ArrayBase::exp2)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(expm1, scalar_expm1_op, exponential of a value minus 1,\sa ArrayBase::expm1)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log, scalar_log_op, natural logarithm,\sa Eigen::log10 DOXCOMMA ArrayBase::log)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log1p, scalar_log1p_op, natural logarithm of 1 plus the value,\sa ArrayBase::log1p)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log10, scalar_log10_op, base 10 logarithm,\sa Eigen::log DOXCOMMA ArrayBase::log10)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(log2, scalar_log2_op, base 2 logarithm,\sa Eigen::log DOXCOMMA ArrayBase::log2)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(abs, scalar_abs_op, absolute value,\sa ArrayBase::abs DOXCOMMA MatrixBase::cwiseAbs)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(abs2, scalar_abs2_op,
+                                 squared absolute value,\sa ArrayBase::abs2 DOXCOMMA MatrixBase::cwiseAbs2)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(arg, scalar_arg_op, complex argument,\sa ArrayBase::arg DOXCOMMA MatrixBase::cwiseArg)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(carg, scalar_carg_op,
+                                 complex argument, \sa ArrayBase::carg DOXCOMMA MatrixBase::cwiseCArg)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sqrt, scalar_sqrt_op, square root,\sa ArrayBase::sqrt DOXCOMMA MatrixBase::cwiseSqrt)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cbrt, scalar_cbrt_op, cube root,\sa ArrayBase::cbrt DOXCOMMA MatrixBase::cwiseCbrt)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(rsqrt, scalar_rsqrt_op, reciprocal square root,\sa ArrayBase::rsqrt)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(square, scalar_square_op,
+                                 square(power 2),\sa Eigen::abs2 DOXCOMMA Eigen::pow DOXCOMMA ArrayBase::square)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(cube, scalar_cube_op, cube(power 3),\sa Eigen::pow DOXCOMMA ArrayBase::cube)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(rint, scalar_rint_op,
+                                 nearest integer,\sa Eigen::floor DOXCOMMA Eigen::ceil DOXCOMMA ArrayBase::round)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(round, scalar_round_op,
+                                 nearest integer,\sa Eigen::floor DOXCOMMA Eigen::ceil DOXCOMMA ArrayBase::round)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(
+    floor, scalar_floor_op, nearest integer not greater than the given value,\sa Eigen::ceil DOXCOMMA ArrayBase::floor)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(
+    ceil, scalar_ceil_op, nearest integer not less than the given value,\sa Eigen::floor DOXCOMMA ArrayBase::ceil)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(trunc, scalar_trunc_op,
+                                 nearest integer not greater in magnitude than the given value,\sa Eigen::trunc DOXCOMMA
+                                     ArrayBase::trunc)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(
+    isnan, scalar_isnan_op, not -a - number test,\sa Eigen::isinf DOXCOMMA Eigen::isfinite DOXCOMMA ArrayBase::isnan)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(
+    isinf, scalar_isinf_op, infinite value test,\sa Eigen::isnan DOXCOMMA Eigen::isfinite DOXCOMMA ArrayBase::isinf)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(isfinite, scalar_isfinite_op,
+                                 finite value test,\sa Eigen::isinf DOXCOMMA Eigen::isnan DOXCOMMA ArrayBase::isfinite)
+EIGEN_ARRAY_DECLARE_GLOBAL_UNARY(sign, scalar_sign_op, sign(or 0),\sa ArrayBase::sign)
+
+template <typename Derived, typename ScalarExponent>
+using GlobalUnaryPowReturnType = std::enable_if_t<
+    !internal::is_arithmetic<typename NumTraits<Derived>::Real>::value &&
+        internal::is_arithmetic<typename NumTraits<ScalarExponent>::Real>::value,
+    CwiseUnaryOp<internal::scalar_unary_pow_op<typename Derived::Scalar, ScalarExponent>, const Derived> >;
+
+/** \returns an expression of the coefficient-wise power of \a x to the given constant \a exponent.
+ *
+ * \tparam ScalarExponent is the scalar type of \a exponent. It must be compatible with the scalar type of the given
+ * expression (\c Derived::Scalar).
+ *
+ * \sa ArrayBase::pow()
+ *
+ * \relates ArrayBase
+ */
+#ifdef EIGEN_PARSED_BY_DOXYGEN
+template <typename Derived, typename ScalarExponent>
+EIGEN_DEVICE_FUNC inline const GlobalUnaryPowReturnType<Derived, ScalarExponent> pow(const Eigen::ArrayBase<Derived>& x,
+                                                                                     const ScalarExponent& exponent);
+#else
+template <typename Derived, typename ScalarExponent>
+EIGEN_DEVICE_FUNC inline const GlobalUnaryPowReturnType<Derived, ScalarExponent> pow(const Eigen::ArrayBase<Derived>& x,
+                                                                                     const ScalarExponent& exponent) {
+  return GlobalUnaryPowReturnType<Derived, ScalarExponent>(
+      x.derived(), internal::scalar_unary_pow_op<typename Derived::Scalar, ScalarExponent>(exponent));
+}
+#endif
+
+/** \returns an expression of the coefficient-wise power of \a x to the given array of \a exponents.
+ *
+ * This function computes the coefficient-wise power.
+ *
+ * Example: \include Cwise_array_power_array.cpp
+ * Output: \verbinclude Cwise_array_power_array.out
+ *
+ * \sa ArrayBase::pow()
+ *
+ * \relates ArrayBase
+ */
+template <typename Derived, typename ExponentDerived>
+inline const Eigen::CwiseBinaryOp<
+    Eigen::internal::scalar_pow_op<typename Derived::Scalar, typename ExponentDerived::Scalar>, const Derived,
+    const ExponentDerived>
+pow(const Eigen::ArrayBase<Derived>& x, const Eigen::ArrayBase<ExponentDerived>& exponents) {
+  return Eigen::CwiseBinaryOp<
+      Eigen::internal::scalar_pow_op<typename Derived::Scalar, typename ExponentDerived::Scalar>, const Derived,
+      const ExponentDerived>(x.derived(), exponents.derived());
+}
+
+/** \returns an expression of the coefficient-wise power of the scalar \a x to the given array of \a exponents.
+ *
+ * This function computes the coefficient-wise power between a scalar and an array of exponents.
+ *
+ * \tparam Scalar is the scalar type of \a x. It must be compatible with the scalar type of the given array expression
+ * (\c Derived::Scalar).
+ *
+ * Example: \include Cwise_scalar_power_array.cpp
+ * Output: \verbinclude Cwise_scalar_power_array.out
+ *
+ * \sa ArrayBase::pow()
+ *
+ * \relates ArrayBase
+ */
+#ifdef EIGEN_PARSED_BY_DOXYGEN
+template <typename Scalar, typename Derived>
+inline const CwiseBinaryOp<internal::scalar_pow_op<Scalar, Derived::Scalar>, Constant<Scalar>, Derived> pow(
+    const Scalar& x, const Eigen::ArrayBase<Derived>& x);
+#else
+template <typename Scalar, typename Derived>
+EIGEN_DEVICE_FUNC inline const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(
+    typename internal::promote_scalar_arg<typename Derived::Scalar EIGEN_COMMA Scalar EIGEN_COMMA
+                                              EIGEN_SCALAR_BINARY_SUPPORTED(pow, Scalar,
+                                                                            typename Derived::Scalar)>::type,
+    Derived, pow) pow(const Scalar& x, const Eigen::ArrayBase<Derived>& exponents) {
+  typedef
+      typename internal::promote_scalar_arg<typename Derived::Scalar, Scalar,
+                                            EIGEN_SCALAR_BINARY_SUPPORTED(pow, Scalar, typename Derived::Scalar)>::type
+          PromotedScalar;
+  return EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(PromotedScalar, Derived, pow)(
+      typename internal::plain_constant_type<Derived, PromotedScalar>::type(
+          exponents.derived().rows(), exponents.derived().cols(), internal::scalar_constant_op<PromotedScalar>(x)),
+      exponents.derived());
+}
+#endif
+
+/** \returns an expression of the coefficient-wise atan2(\a x, \a y). \a x and \a y must be of the same type.
+ *
+ * This function computes the coefficient-wise atan2().
+ *
+ * \sa ArrayBase::atan2()
+ *
+ * \relates ArrayBase
+ */
+template <typename LhsDerived, typename RhsDerived>
+inline const std::enable_if_t<
+    std::is_same<typename LhsDerived::Scalar, typename RhsDerived::Scalar>::value,
+    Eigen::CwiseBinaryOp<Eigen::internal::scalar_atan2_op<typename LhsDerived::Scalar, typename RhsDerived::Scalar>,
+                         const LhsDerived, const RhsDerived> >
+atan2(const Eigen::ArrayBase<LhsDerived>& x, const Eigen::ArrayBase<RhsDerived>& exponents) {
+  return Eigen::CwiseBinaryOp<
+      Eigen::internal::scalar_atan2_op<typename LhsDerived::Scalar, typename RhsDerived::Scalar>, const LhsDerived,
+      const RhsDerived>(x.derived(), exponents.derived());
+}
+
+namespace internal {
+EIGEN_ARRAY_DECLARE_GLOBAL_EIGEN_UNARY(real, scalar_real_op)
+EIGEN_ARRAY_DECLARE_GLOBAL_EIGEN_UNARY(imag, scalar_imag_op)
+EIGEN_ARRAY_DECLARE_GLOBAL_EIGEN_UNARY(abs2, scalar_abs2_op)
+}  // namespace internal
+}  // namespace Eigen
+
+// TODO: cleanly disable those functions that are not supported on Array (numext::real_ref, internal::random,
+// internal::isApprox...)
+
+#endif  // EIGEN_GLOBAL_FUNCTIONS_H

include/Eigen/src/Core/IO.h

@@ -0,0 +1,233 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_IO_H
+#define EIGEN_IO_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+enum { DontAlignCols = 1 };
+enum { StreamPrecision = -1, FullPrecision = -2 };
+
+namespace internal {
+template <typename Derived>
+std::ostream& print_matrix(std::ostream& s, const Derived& _m, const IOFormat& fmt);
+}
+
+/** \class IOFormat
+ * \ingroup Core_Module
+ *
+ * \brief Stores a set of parameters controlling the way matrices are printed
+ *
+ * List of available parameters:
+ *  - \b precision number of digits for floating point values, or one of the special constants \c StreamPrecision and \c
+ * FullPrecision. The default is the special value \c StreamPrecision which means to use the stream's own precision
+ * setting, as set for instance using \c cout.precision(3). The other special value \c FullPrecision means that the
+ * number of digits will be computed to match the full precision of each floating-point type.
+ *  - \b flags an OR-ed combination of flags, the default value is 0, the only currently available flag is \c
+ * DontAlignCols which allows to disable the alignment of columns, resulting in faster code.
+ *  - \b coeffSeparator string printed between two coefficients of the same row
+ *  - \b rowSeparator string printed between two rows
+ *  - \b rowPrefix string printed at the beginning of each row
+ *  - \b rowSuffix string printed at the end of each row
+ *  - \b matPrefix string printed at the beginning of the matrix
+ *  - \b matSuffix string printed at the end of the matrix
+ *  - \b fill character printed to fill the empty space in aligned columns
+ *
+ * Example: \include IOFormat.cpp
+ * Output: \verbinclude IOFormat.out
+ *
+ * \sa DenseBase::format(), class WithFormat
+ */
+struct IOFormat {
+  /** Default constructor, see class IOFormat for the meaning of the parameters */
+  IOFormat(int _precision = StreamPrecision, int _flags = 0, const std::string& _coeffSeparator = " ",
+           const std::string& _rowSeparator = "\n", const std::string& _rowPrefix = "",
+           const std::string& _rowSuffix = "", const std::string& _matPrefix = "", const std::string& _matSuffix = "",
+           const char _fill = ' ')
+      : matPrefix(_matPrefix),
+        matSuffix(_matSuffix),
+        rowPrefix(_rowPrefix),
+        rowSuffix(_rowSuffix),
+        rowSeparator(_rowSeparator),
+        rowSpacer(""),
+        coeffSeparator(_coeffSeparator),
+        fill(_fill),
+        precision(_precision),
+        flags(_flags) {
+    // TODO check if rowPrefix, rowSuffix or rowSeparator contains a newline
+    // don't add rowSpacer if columns are not to be aligned
+    if ((flags & DontAlignCols)) return;
+    int i = int(matPrefix.length()) - 1;
+    while (i >= 0 && matPrefix[i] != '\n') {
+      rowSpacer += ' ';
+      i--;
+    }
+  }
+  std::string matPrefix, matSuffix;
+  std::string rowPrefix, rowSuffix, rowSeparator, rowSpacer;
+  std::string coeffSeparator;
+  char fill;
+  int precision;
+  int flags;
+};
+
+/** \class WithFormat
+ * \ingroup Core_Module
+ *
+ * \brief Pseudo expression providing matrix output with given format
+ *
+ * \tparam ExpressionType the type of the object on which IO stream operations are performed
+ *
+ * This class represents an expression with stream operators controlled by a given IOFormat.
+ * It is the return type of DenseBase::format()
+ * and most of the time this is the only way it is used.
+ *
+ * See class IOFormat for some examples.
+ *
+ * \sa DenseBase::format(), class IOFormat
+ */
+template <typename ExpressionType>
+class WithFormat {
+ public:
+  WithFormat(const ExpressionType& matrix, const IOFormat& format) : m_matrix(matrix), m_format(format) {}
+
+  friend std::ostream& operator<<(std::ostream& s, const WithFormat& wf) {
+    return internal::print_matrix(s, wf.m_matrix.eval(), wf.m_format);
+  }
+
+ protected:
+  typename ExpressionType::Nested m_matrix;
+  IOFormat m_format;
+};
+
+namespace internal {
+
+// NOTE: This helper is kept for backward compatibility with previous code specializing
+//       this internal::significant_decimals_impl structure. In the future we should directly
+//       call max_digits10().
+template <typename Scalar>
+struct significant_decimals_impl {
+  static inline int run() { return NumTraits<Scalar>::max_digits10(); }
+};
+
+/** \internal
+ * print the matrix \a _m to the output stream \a s using the output format \a fmt */
+template <typename Derived>
+std::ostream& print_matrix(std::ostream& s, const Derived& _m, const IOFormat& fmt) {
+  using internal::is_same;
+
+  if (_m.size() == 0) {
+    s << fmt.matPrefix << fmt.matSuffix;
+    return s;
+  }
+
+  typename Derived::Nested m = _m;
+  typedef typename Derived::Scalar Scalar;
+  typedef std::conditional_t<is_same<Scalar, char>::value || is_same<Scalar, unsigned char>::value ||
+                                 is_same<Scalar, numext::int8_t>::value || is_same<Scalar, numext::uint8_t>::value,
+                             int,
+                             std::conditional_t<is_same<Scalar, std::complex<char> >::value ||
+                                                    is_same<Scalar, std::complex<unsigned char> >::value ||
+                                                    is_same<Scalar, std::complex<numext::int8_t> >::value ||
+                                                    is_same<Scalar, std::complex<numext::uint8_t> >::value,
+                                                std::complex<int>, const Scalar&> >
+      PrintType;
+
+  Index width = 0;
+
+  std::streamsize explicit_precision;
+  if (fmt.precision == StreamPrecision) {
+    explicit_precision = 0;
+  } else if (fmt.precision == FullPrecision) {
+    if (NumTraits<Scalar>::IsInteger) {
+      explicit_precision = 0;
+    } else {
+      explicit_precision = significant_decimals_impl<Scalar>::run();
+    }
+  } else {
+    explicit_precision = fmt.precision;
+  }
+
+  std::streamsize old_precision = 0;
+  if (explicit_precision) old_precision = s.precision(explicit_precision);
+
+  bool align_cols = !(fmt.flags & DontAlignCols);
+  if (align_cols) {
+    // compute the largest width
+    for (Index j = 0; j < m.cols(); ++j)
+      for (Index i = 0; i < m.rows(); ++i) {
+        std::stringstream sstr;
+        sstr.copyfmt(s);
+        sstr << static_cast<PrintType>(m.coeff(i, j));
+        width = std::max<Index>(width, Index(sstr.str().length()));
+      }
+  }
+  std::streamsize old_width = s.width();
+  char old_fill_character = s.fill();
+  s << fmt.matPrefix;
+  for (Index i = 0; i < m.rows(); ++i) {
+    if (i) s << fmt.rowSpacer;
+    s << fmt.rowPrefix;
+    if (width) {
+      s.fill(fmt.fill);
+      s.width(width);
+    }
+    s << static_cast<PrintType>(m.coeff(i, 0));
+    for (Index j = 1; j < m.cols(); ++j) {
+      s << fmt.coeffSeparator;
+      if (width) {
+        s.fill(fmt.fill);
+        s.width(width);
+      }
+      s << static_cast<PrintType>(m.coeff(i, j));
+    }
+    s << fmt.rowSuffix;
+    if (i < m.rows() - 1) s << fmt.rowSeparator;
+  }
+  s << fmt.matSuffix;
+  if (explicit_precision) s.precision(old_precision);
+  if (width) {
+    s.fill(old_fill_character);
+    s.width(old_width);
+  }
+  return s;
+}
+
+}  // end namespace internal
+
+/** \relates DenseBase
+ *
+ * Outputs the matrix, to the given stream.
+ *
+ * If you wish to print the matrix with a format different than the default, use DenseBase::format().
+ *
+ * It is also possible to change the default format by defining EIGEN_DEFAULT_IO_FORMAT before including Eigen headers.
+ * If not defined, this will automatically be defined to Eigen::IOFormat(), that is the Eigen::IOFormat with default
+ * parameters.
+ *
+ * \sa DenseBase::format()
+ */
+template <typename Derived>
+std::ostream& operator<<(std::ostream& s, const DenseBase<Derived>& m) {
+  return internal::print_matrix(s, m.eval(), EIGEN_DEFAULT_IO_FORMAT);
+}
+
+template <typename Derived>
+std::ostream& operator<<(std::ostream& s, const DiagonalBase<Derived>& m) {
+  return internal::print_matrix(s, m.derived(), EIGEN_DEFAULT_IO_FORMAT);
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_IO_H

include/Eigen/src/Core/IndexedView.h

@@ -0,0 +1,315 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_INDEXED_VIEW_H
+#define EIGEN_INDEXED_VIEW_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename XprType, typename RowIndices, typename ColIndices>
+struct traits<IndexedView<XprType, RowIndices, ColIndices>> : traits<XprType> {
+  enum {
+    RowsAtCompileTime = int(IndexedViewHelper<RowIndices>::SizeAtCompileTime),
+    ColsAtCompileTime = int(IndexedViewHelper<ColIndices>::SizeAtCompileTime),
+    MaxRowsAtCompileTime = RowsAtCompileTime,
+    MaxColsAtCompileTime = ColsAtCompileTime,
+
+    XprTypeIsRowMajor = (int(traits<XprType>::Flags) & RowMajorBit) != 0,
+    IsRowMajor = (MaxRowsAtCompileTime == 1 && MaxColsAtCompileTime != 1)   ? 1
+                 : (MaxColsAtCompileTime == 1 && MaxRowsAtCompileTime != 1) ? 0
+                                                                            : XprTypeIsRowMajor,
+
+    RowIncr = int(IndexedViewHelper<RowIndices>::IncrAtCompileTime),
+    ColIncr = int(IndexedViewHelper<ColIndices>::IncrAtCompileTime),
+    InnerIncr = IsRowMajor ? ColIncr : RowIncr,
+    OuterIncr = IsRowMajor ? RowIncr : ColIncr,
+
+    HasSameStorageOrderAsXprType = (IsRowMajor == XprTypeIsRowMajor),
+    XprInnerStride = HasSameStorageOrderAsXprType ? int(inner_stride_at_compile_time<XprType>::ret)
+                                                  : int(outer_stride_at_compile_time<XprType>::ret),
+    XprOuterstride = HasSameStorageOrderAsXprType ? int(outer_stride_at_compile_time<XprType>::ret)
+                                                  : int(inner_stride_at_compile_time<XprType>::ret),
+
+    InnerSize = XprTypeIsRowMajor ? ColsAtCompileTime : RowsAtCompileTime,
+    IsBlockAlike = InnerIncr == 1 && OuterIncr == 1,
+    IsInnerPannel = HasSameStorageOrderAsXprType &&
+                    is_same<AllRange<InnerSize>, std::conditional_t<XprTypeIsRowMajor, ColIndices, RowIndices>>::value,
+
+    InnerStrideAtCompileTime =
+        InnerIncr < 0 || InnerIncr == DynamicIndex || XprInnerStride == Dynamic || InnerIncr == Undefined
+            ? Dynamic
+            : XprInnerStride * InnerIncr,
+    OuterStrideAtCompileTime =
+        OuterIncr < 0 || OuterIncr == DynamicIndex || XprOuterstride == Dynamic || OuterIncr == Undefined
+            ? Dynamic
+            : XprOuterstride * OuterIncr,
+
+    ReturnAsScalar = is_single_range<RowIndices>::value && is_single_range<ColIndices>::value,
+    ReturnAsBlock = (!ReturnAsScalar) && IsBlockAlike,
+    ReturnAsIndexedView = (!ReturnAsScalar) && (!ReturnAsBlock),
+
+    // FIXME we deal with compile-time strides if and only if we have DirectAccessBit flag,
+    // but this is too strict regarding negative strides...
+    DirectAccessMask = (int(InnerIncr) != Undefined && int(OuterIncr) != Undefined && InnerIncr >= 0 && OuterIncr >= 0)
+                           ? DirectAccessBit
+                           : 0,
+    FlagsRowMajorBit = IsRowMajor ? RowMajorBit : 0,
+    FlagsLvalueBit = is_lvalue<XprType>::value ? LvalueBit : 0,
+    FlagsLinearAccessBit = (RowsAtCompileTime == 1 || ColsAtCompileTime == 1) ? LinearAccessBit : 0,
+    Flags = (traits<XprType>::Flags & (HereditaryBits | DirectAccessMask)) | FlagsLvalueBit | FlagsRowMajorBit |
+            FlagsLinearAccessBit
+  };
+
+  typedef Block<XprType, RowsAtCompileTime, ColsAtCompileTime, IsInnerPannel> BlockType;
+};
+
+template <typename XprType, typename RowIndices, typename ColIndices, typename StorageKind, bool DirectAccess>
+class IndexedViewImpl;
+
+}  // namespace internal
+
+/** \class IndexedView
+ * \ingroup Core_Module
+ *
+ * \brief Expression of a non-sequential sub-matrix defined by arbitrary sequences of row and column indices
+ *
+ * \tparam XprType the type of the expression in which we are taking the intersections of sub-rows and sub-columns
+ * \tparam RowIndices the type of the object defining the sequence of row indices
+ * \tparam ColIndices the type of the object defining the sequence of column indices
+ *
+ * This class represents an expression of a sub-matrix (or sub-vector) defined as the intersection
+ * of sub-sets of rows and columns, that are themself defined by generic sequences of row indices \f$
+ * \{r_0,r_1,..r_{m-1}\} \f$ and column indices \f$ \{c_0,c_1,..c_{n-1} \}\f$. Let \f$ A \f$  be the nested matrix, then
+ * the resulting matrix \f$ B \f$ has \c m rows and \c n columns, and its entries are given by: \f$ B(i,j) = A(r_i,c_j)
+ * \f$.
+ *
+ * The \c RowIndices and \c ColIndices types must be compatible with the following API:
+ * \code
+ * <integral type> operator[](Index) const;
+ * Index size() const;
+ * \endcode
+ *
+ * Typical supported types thus include:
+ *  - std::vector<int>
+ *  - std::valarray<int>
+ *  - std::array<int>
+ *  - Eigen::ArrayXi
+ *  - decltype(ArrayXi::LinSpaced(...))
+ *  - Any view/expressions of the previous types
+ *  - Eigen::ArithmeticSequence
+ *  - Eigen::internal::AllRange     (helper for Eigen::placeholders::all)
+ *  - Eigen::internal::SingleRange  (helper for single index)
+ *  - etc.
+ *
+ * In typical usages of %Eigen, this class should never be used directly. It is the return type of
+ * DenseBase::operator()(const RowIndices&, const ColIndices&).
+ *
+ * \sa class Block
+ */
+template <typename XprType, typename RowIndices, typename ColIndices>
+class IndexedView
+    : public internal::IndexedViewImpl<XprType, RowIndices, ColIndices, typename internal::traits<XprType>::StorageKind,
+                                       (internal::traits<IndexedView<XprType, RowIndices, ColIndices>>::Flags &
+                                        DirectAccessBit) != 0> {
+ public:
+  typedef typename internal::IndexedViewImpl<
+      XprType, RowIndices, ColIndices, typename internal::traits<XprType>::StorageKind,
+      (internal::traits<IndexedView<XprType, RowIndices, ColIndices>>::Flags & DirectAccessBit) != 0>
+      Base;
+  EIGEN_GENERIC_PUBLIC_INTERFACE(IndexedView)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(IndexedView)
+
+  template <typename T0, typename T1>
+  IndexedView(XprType& xpr, const T0& rowIndices, const T1& colIndices) : Base(xpr, rowIndices, colIndices) {}
+};
+
+namespace internal {
+
+// Generic API dispatcher
+template <typename XprType, typename RowIndices, typename ColIndices, typename StorageKind, bool DirectAccess>
+class IndexedViewImpl : public internal::generic_xpr_base<IndexedView<XprType, RowIndices, ColIndices>>::type {
+ public:
+  typedef typename internal::generic_xpr_base<IndexedView<XprType, RowIndices, ColIndices>>::type Base;
+  typedef typename internal::ref_selector<XprType>::non_const_type MatrixTypeNested;
+  typedef internal::remove_all_t<XprType> NestedExpression;
+  typedef typename XprType::Scalar Scalar;
+
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(IndexedViewImpl)
+
+  template <typename T0, typename T1>
+  IndexedViewImpl(XprType& xpr, const T0& rowIndices, const T1& colIndices)
+      : m_xpr(xpr), m_rowIndices(rowIndices), m_colIndices(colIndices) {}
+
+  /** \returns number of rows */
+  Index rows() const { return IndexedViewHelper<RowIndices>::size(m_rowIndices); }
+
+  /** \returns number of columns */
+  Index cols() const { return IndexedViewHelper<ColIndices>::size(m_colIndices); }
+
+  /** \returns the nested expression */
+  const internal::remove_all_t<XprType>& nestedExpression() const { return m_xpr; }
+
+  /** \returns the nested expression */
+  std::remove_reference_t<XprType>& nestedExpression() { return m_xpr; }
+
+  /** \returns a const reference to the object storing/generating the row indices */
+  const RowIndices& rowIndices() const { return m_rowIndices; }
+
+  /** \returns a const reference to the object storing/generating the column indices */
+  const ColIndices& colIndices() const { return m_colIndices; }
+
+  constexpr Scalar& coeffRef(Index rowId, Index colId) {
+    return nestedExpression().coeffRef(m_rowIndices[rowId], m_colIndices[colId]);
+  }
+
+  constexpr const Scalar& coeffRef(Index rowId, Index colId) const {
+    return nestedExpression().coeffRef(m_rowIndices[rowId], m_colIndices[colId]);
+  }
+
+ protected:
+  MatrixTypeNested m_xpr;
+  RowIndices m_rowIndices;
+  ColIndices m_colIndices;
+};
+
+template <typename XprType, typename RowIndices, typename ColIndices, typename StorageKind>
+class IndexedViewImpl<XprType, RowIndices, ColIndices, StorageKind, true>
+    : public IndexedViewImpl<XprType, RowIndices, ColIndices, StorageKind, false> {
+ public:
+  using Base = internal::IndexedViewImpl<XprType, RowIndices, ColIndices,
+                                         typename internal::traits<XprType>::StorageKind, false>;
+  using Derived = IndexedView<XprType, RowIndices, ColIndices>;
+
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(IndexedViewImpl)
+
+  template <typename T0, typename T1>
+  IndexedViewImpl(XprType& xpr, const T0& rowIndices, const T1& colIndices) : Base(xpr, rowIndices, colIndices) {}
+
+  Index rowIncrement() const {
+    if (traits<Derived>::RowIncr != DynamicIndex && traits<Derived>::RowIncr != Undefined) {
+      return traits<Derived>::RowIncr;
+    }
+    return IndexedViewHelper<RowIndices>::incr(this->rowIndices());
+  }
+  Index colIncrement() const {
+    if (traits<Derived>::ColIncr != DynamicIndex && traits<Derived>::ColIncr != Undefined) {
+      return traits<Derived>::ColIncr;
+    }
+    return IndexedViewHelper<ColIndices>::incr(this->colIndices());
+  }
+
+  Index innerIncrement() const { return traits<Derived>::IsRowMajor ? colIncrement() : rowIncrement(); }
+
+  Index outerIncrement() const { return traits<Derived>::IsRowMajor ? rowIncrement() : colIncrement(); }
+
+  std::decay_t<typename XprType::Scalar>* data() {
+    Index row_offset = this->rowIndices()[0] * this->nestedExpression().rowStride();
+    Index col_offset = this->colIndices()[0] * this->nestedExpression().colStride();
+    return this->nestedExpression().data() + row_offset + col_offset;
+  }
+
+  const std::decay_t<typename XprType::Scalar>* data() const {
+    Index row_offset = this->rowIndices()[0] * this->nestedExpression().rowStride();
+    Index col_offset = this->colIndices()[0] * this->nestedExpression().colStride();
+    return this->nestedExpression().data() + row_offset + col_offset;
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT {
+    if (traits<Derived>::InnerStrideAtCompileTime != Dynamic) {
+      return traits<Derived>::InnerStrideAtCompileTime;
+    }
+    return innerIncrement() * this->nestedExpression().innerStride();
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT {
+    if (traits<Derived>::OuterStrideAtCompileTime != Dynamic) {
+      return traits<Derived>::OuterStrideAtCompileTime;
+    }
+    return outerIncrement() * this->nestedExpression().outerStride();
+  }
+};
+
+template <typename ArgType, typename RowIndices, typename ColIndices>
+struct unary_evaluator<IndexedView<ArgType, RowIndices, ColIndices>, IndexBased>
+    : evaluator_base<IndexedView<ArgType, RowIndices, ColIndices>> {
+  typedef IndexedView<ArgType, RowIndices, ColIndices> XprType;
+
+  enum {
+    CoeffReadCost = evaluator<ArgType>::CoeffReadCost /* TODO + cost of row/col index */,
+
+    FlagsLinearAccessBit =
+        (traits<XprType>::RowsAtCompileTime == 1 || traits<XprType>::ColsAtCompileTime == 1) ? LinearAccessBit : 0,
+
+    FlagsRowMajorBit = traits<XprType>::FlagsRowMajorBit,
+
+    Flags = (evaluator<ArgType>::Flags & (HereditaryBits & ~RowMajorBit /*| LinearAccessBit | DirectAccessBit*/)) |
+            FlagsLinearAccessBit | FlagsRowMajorBit,
+
+    Alignment = 0
+  };
+
+  EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& xpr) : m_argImpl(xpr.nestedExpression()), m_xpr(xpr) {
+    EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
+  }
+
+  typedef typename XprType::Scalar Scalar;
+  typedef typename XprType::CoeffReturnType CoeffReturnType;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
+    eigen_assert(m_xpr.rowIndices()[row] >= 0 && m_xpr.rowIndices()[row] < m_xpr.nestedExpression().rows() &&
+                 m_xpr.colIndices()[col] >= 0 && m_xpr.colIndices()[col] < m_xpr.nestedExpression().cols());
+    return m_argImpl.coeff(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) {
+    eigen_assert(m_xpr.rowIndices()[row] >= 0 && m_xpr.rowIndices()[row] < m_xpr.nestedExpression().rows() &&
+                 m_xpr.colIndices()[col] >= 0 && m_xpr.colIndices()[col] < m_xpr.nestedExpression().cols());
+    return m_argImpl.coeffRef(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) {
+    EIGEN_STATIC_ASSERT_LVALUE(XprType)
+    Index row = XprType::RowsAtCompileTime == 1 ? 0 : index;
+    Index col = XprType::RowsAtCompileTime == 1 ? index : 0;
+    eigen_assert(m_xpr.rowIndices()[row] >= 0 && m_xpr.rowIndices()[row] < m_xpr.nestedExpression().rows() &&
+                 m_xpr.colIndices()[col] >= 0 && m_xpr.colIndices()[col] < m_xpr.nestedExpression().cols());
+    return m_argImpl.coeffRef(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeffRef(Index index) const {
+    Index row = XprType::RowsAtCompileTime == 1 ? 0 : index;
+    Index col = XprType::RowsAtCompileTime == 1 ? index : 0;
+    eigen_assert(m_xpr.rowIndices()[row] >= 0 && m_xpr.rowIndices()[row] < m_xpr.nestedExpression().rows() &&
+                 m_xpr.colIndices()[col] >= 0 && m_xpr.colIndices()[col] < m_xpr.nestedExpression().cols());
+    return m_argImpl.coeffRef(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CoeffReturnType coeff(Index index) const {
+    Index row = XprType::RowsAtCompileTime == 1 ? 0 : index;
+    Index col = XprType::RowsAtCompileTime == 1 ? index : 0;
+    eigen_assert(m_xpr.rowIndices()[row] >= 0 && m_xpr.rowIndices()[row] < m_xpr.nestedExpression().rows() &&
+                 m_xpr.colIndices()[col] >= 0 && m_xpr.colIndices()[col] < m_xpr.nestedExpression().cols());
+    return m_argImpl.coeff(m_xpr.rowIndices()[row], m_xpr.colIndices()[col]);
+  }
+
+ protected:
+  evaluator<ArgType> m_argImpl;
+  const XprType& m_xpr;
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_INDEXED_VIEW_H

include/Eigen/src/Core/InnerProduct.h

@@ -0,0 +1,250 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2024 Charlie Schlosser <cs.schlosser@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_INNER_PRODUCT_EVAL_H
+#define EIGEN_INNER_PRODUCT_EVAL_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+// recursively searches for the largest simd type that does not exceed Size, or the smallest if no such type exists
+template <typename Scalar, int Size, typename Packet = typename packet_traits<Scalar>::type,
+          bool Stop =
+              (unpacket_traits<Packet>::size <= Size) || is_same<Packet, typename unpacket_traits<Packet>::half>::value>
+struct find_inner_product_packet_helper;
+
+template <typename Scalar, int Size, typename Packet>
+struct find_inner_product_packet_helper<Scalar, Size, Packet, false> {
+  using type = typename find_inner_product_packet_helper<Scalar, Size, typename unpacket_traits<Packet>::half>::type;
+};
+
+template <typename Scalar, int Size, typename Packet>
+struct find_inner_product_packet_helper<Scalar, Size, Packet, true> {
+  using type = Packet;
+};
+
+template <typename Scalar, int Size>
+struct find_inner_product_packet : find_inner_product_packet_helper<Scalar, Size> {};
+
+template <typename Scalar>
+struct find_inner_product_packet<Scalar, Dynamic> {
+  using type = typename packet_traits<Scalar>::type;
+};
+
+template <typename Lhs, typename Rhs>
+struct inner_product_assert {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Lhs)
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Rhs)
+  EIGEN_STATIC_ASSERT_SAME_VECTOR_SIZE(Lhs, Rhs)
+#ifndef EIGEN_NO_DEBUG
+  static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, const Rhs& rhs) {
+    eigen_assert((lhs.size() == rhs.size()) && "Inner product: lhs and rhs vectors must have same size");
+  }
+#else
+  static EIGEN_DEVICE_FUNC void run(const Lhs&, const Rhs&) {}
+#endif
+};
+
+template <typename Func, typename Lhs, typename Rhs>
+struct inner_product_evaluator {
+  static constexpr int LhsFlags = evaluator<Lhs>::Flags, RhsFlags = evaluator<Rhs>::Flags,
+                       SizeAtCompileTime = min_size_prefer_fixed(Lhs::SizeAtCompileTime, Rhs::SizeAtCompileTime),
+                       LhsAlignment = evaluator<Lhs>::Alignment, RhsAlignment = evaluator<Rhs>::Alignment;
+
+  using Scalar = typename Func::result_type;
+  using Packet = typename find_inner_product_packet<Scalar, SizeAtCompileTime>::type;
+
+  static constexpr bool Vectorize =
+      bool(LhsFlags & RhsFlags & PacketAccessBit) && Func::PacketAccess &&
+      ((SizeAtCompileTime == Dynamic) || (unpacket_traits<Packet>::size <= SizeAtCompileTime));
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit inner_product_evaluator(const Lhs& lhs, const Rhs& rhs,
+                                                                         Func func = Func())
+      : m_func(func), m_lhs(lhs), m_rhs(rhs), m_size(lhs.size()) {
+    inner_product_assert<Lhs, Rhs>::run(lhs, rhs);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index size() const { return m_size.value(); }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index index) const {
+    return m_func.coeff(m_lhs.coeff(index), m_rhs.coeff(index));
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& value, Index index) const {
+    return m_func.coeff(value, m_lhs.coeff(index), m_rhs.coeff(index));
+  }
+
+  template <typename PacketType, int LhsMode = LhsAlignment, int RhsMode = RhsAlignment>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const {
+    return m_func.packet(m_lhs.template packet<LhsMode, PacketType>(index),
+                         m_rhs.template packet<RhsMode, PacketType>(index));
+  }
+
+  template <typename PacketType, int LhsMode = LhsAlignment, int RhsMode = RhsAlignment>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(const PacketType& value, Index index) const {
+    return m_func.packet(value, m_lhs.template packet<LhsMode, PacketType>(index),
+                         m_rhs.template packet<RhsMode, PacketType>(index));
+  }
+
+  const Func m_func;
+  const evaluator<Lhs> m_lhs;
+  const evaluator<Rhs> m_rhs;
+  const variable_if_dynamic<Index, SizeAtCompileTime> m_size;
+};
+
+template <typename Evaluator, bool Vectorize = Evaluator::Vectorize>
+struct inner_product_impl;
+
+// scalar loop
+template <typename Evaluator>
+struct inner_product_impl<Evaluator, false> {
+  using Scalar = typename Evaluator::Scalar;
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval) {
+    const Index size = eval.size();
+    if (size == 0) return Scalar(0);
+
+    Scalar result = eval.coeff(0);
+    for (Index k = 1; k < size; k++) {
+      result = eval.coeff(result, k);
+    }
+
+    return result;
+  }
+};
+
+// vector loop
+template <typename Evaluator>
+struct inner_product_impl<Evaluator, true> {
+  using UnsignedIndex = std::make_unsigned_t<Index>;
+  using Scalar = typename Evaluator::Scalar;
+  using Packet = typename Evaluator::Packet;
+  static constexpr int PacketSize = unpacket_traits<Packet>::size;
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval) {
+    const UnsignedIndex size = static_cast<UnsignedIndex>(eval.size());
+    if (size < PacketSize) return inner_product_impl<Evaluator, false>::run(eval);
+
+    const UnsignedIndex packetEnd = numext::round_down(size, PacketSize);
+    const UnsignedIndex quadEnd = numext::round_down(size, 4 * PacketSize);
+    const UnsignedIndex numPackets = size / PacketSize;
+    const UnsignedIndex numRemPackets = (packetEnd - quadEnd) / PacketSize;
+
+    Packet presult0, presult1, presult2, presult3;
+
+    presult0 = eval.template packet<Packet>(0 * PacketSize);
+    if (numPackets >= 2) presult1 = eval.template packet<Packet>(1 * PacketSize);
+    if (numPackets >= 3) presult2 = eval.template packet<Packet>(2 * PacketSize);
+    if (numPackets >= 4) {
+      presult3 = eval.template packet<Packet>(3 * PacketSize);
+
+      for (UnsignedIndex k = 4 * PacketSize; k < quadEnd; k += 4 * PacketSize) {
+        presult0 = eval.packet(presult0, k + 0 * PacketSize);
+        presult1 = eval.packet(presult1, k + 1 * PacketSize);
+        presult2 = eval.packet(presult2, k + 2 * PacketSize);
+        presult3 = eval.packet(presult3, k + 3 * PacketSize);
+      }
+
+      if (numRemPackets >= 1) presult0 = eval.packet(presult0, quadEnd + 0 * PacketSize);
+      if (numRemPackets >= 2) presult1 = eval.packet(presult1, quadEnd + 1 * PacketSize);
+      if (numRemPackets == 3) presult2 = eval.packet(presult2, quadEnd + 2 * PacketSize);
+
+      presult2 = padd(presult2, presult3);
+    }
+
+    if (numPackets >= 3) presult1 = padd(presult1, presult2);
+    if (numPackets >= 2) presult0 = padd(presult0, presult1);
+
+    Scalar result = predux(presult0);
+    for (UnsignedIndex k = packetEnd; k < size; k++) {
+      result = eval.coeff(result, k);
+    }
+
+    return result;
+  }
+};
+
+template <typename Scalar, bool Conj>
+struct conditional_conj;
+
+template <typename Scalar>
+struct conditional_conj<Scalar, true> {
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& a) { return numext::conj(a); }
+  template <typename Packet>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(const Packet& a) {
+    return pconj(a);
+  }
+};
+
+template <typename Scalar>
+struct conditional_conj<Scalar, false> {
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& a) { return a; }
+  template <typename Packet>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(const Packet& a) {
+    return a;
+  }
+};
+
+template <typename LhsScalar, typename RhsScalar, bool Conj>
+struct scalar_inner_product_op {
+  using result_type = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
+  using conj_helper = conditional_conj<LhsScalar, Conj>;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type coeff(const LhsScalar& a, const RhsScalar& b) const {
+    return (conj_helper::coeff(a) * b);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type coeff(const result_type& accum, const LhsScalar& a,
+                                                          const RhsScalar& b) const {
+    return (conj_helper::coeff(a) * b) + accum;
+  }
+  static constexpr bool PacketAccess = false;
+};
+
+template <typename Scalar, bool Conj>
+struct scalar_inner_product_op<Scalar, Scalar, Conj> {
+  using result_type = Scalar;
+  using conj_helper = conditional_conj<Scalar, Conj>;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& a, const Scalar& b) const {
+    return pmul(conj_helper::coeff(a), b);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& accum, const Scalar& a, const Scalar& b) const {
+    return pmadd(conj_helper::coeff(a), b, accum);
+  }
+  template <typename Packet>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(const Packet& a, const Packet& b) const {
+    return pmul(conj_helper::packet(a), b);
+  }
+  template <typename Packet>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(const Packet& accum, const Packet& a, const Packet& b) const {
+    return pmadd(conj_helper::packet(a), b, accum);
+  }
+  static constexpr bool PacketAccess = packet_traits<Scalar>::HasMul && packet_traits<Scalar>::HasAdd;
+};
+
+template <typename Lhs, typename Rhs, bool Conj>
+struct default_inner_product_impl {
+  using LhsScalar = typename traits<Lhs>::Scalar;
+  using RhsScalar = typename traits<Rhs>::Scalar;
+  using Op = scalar_inner_product_op<LhsScalar, RhsScalar, Conj>;
+  using Evaluator = inner_product_evaluator<Op, Lhs, Rhs>;
+  using result_type = typename Evaluator::Scalar;
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type run(const MatrixBase<Lhs>& a, const MatrixBase<Rhs>& b) {
+    Evaluator eval(a.derived(), b.derived(), Op());
+    return inner_product_impl<Evaluator>::run(eval);
+  }
+};
+
+template <typename Lhs, typename Rhs>
+struct dot_impl : default_inner_product_impl<Lhs, Rhs, true> {};
+
+}  // namespace internal
+}  // namespace Eigen
+
+#endif  // EIGEN_INNER_PRODUCT_EVAL_H

include/Eigen/src/Core/InternalHeaderCheck.h

@@ -0,0 +1,3 @@
+#ifndef EIGEN_CORE_MODULE_H
+#error "Please include Eigen/Core instead of including headers inside the src directory directly."
+#endif

include/Eigen/src/Core/Inverse.h

@@ -0,0 +1,108 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2014-2019 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_INVERSE_H
+#define EIGEN_INVERSE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+template <typename XprType, typename StorageKind>
+class InverseImpl;
+
+namespace internal {
+
+template <typename XprType>
+struct traits<Inverse<XprType> > : traits<typename XprType::PlainObject> {
+  typedef typename XprType::PlainObject PlainObject;
+  typedef traits<PlainObject> BaseTraits;
+  enum { Flags = BaseTraits::Flags & RowMajorBit };
+};
+
+}  // end namespace internal
+
+/** \class Inverse
+ *
+ * \brief Expression of the inverse of another expression
+ *
+ * \tparam XprType the type of the expression we are taking the inverse
+ *
+ * This class represents an abstract expression of A.inverse()
+ * and most of the time this is the only way it is used.
+ *
+ */
+template <typename XprType>
+class Inverse : public InverseImpl<XprType, typename internal::traits<XprType>::StorageKind> {
+ public:
+  typedef typename XprType::StorageIndex StorageIndex;
+  typedef typename XprType::Scalar Scalar;
+  typedef typename internal::ref_selector<XprType>::type XprTypeNested;
+  typedef internal::remove_all_t<XprTypeNested> XprTypeNestedCleaned;
+  typedef typename internal::ref_selector<Inverse>::type Nested;
+  typedef internal::remove_all_t<XprType> NestedExpression;
+
+  explicit EIGEN_DEVICE_FUNC Inverse(const XprType& xpr) : m_xpr(xpr) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_xpr.cols(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_xpr.rows(); }
+
+  EIGEN_DEVICE_FUNC const XprTypeNestedCleaned& nestedExpression() const { return m_xpr; }
+
+ protected:
+  XprTypeNested m_xpr;
+};
+
+// Generic API dispatcher
+template <typename XprType, typename StorageKind>
+class InverseImpl : public internal::generic_xpr_base<Inverse<XprType> >::type {
+ public:
+  typedef typename internal::generic_xpr_base<Inverse<XprType> >::type Base;
+  typedef typename XprType::Scalar Scalar;
+
+ private:
+  Scalar coeff(Index row, Index col) const;
+  Scalar coeff(Index i) const;
+};
+
+namespace internal {
+
+/** \internal
+ * \brief Default evaluator for Inverse expression.
+ *
+ * This default evaluator for Inverse expression simply evaluate the inverse into a temporary
+ * by a call to internal::call_assignment_no_alias.
+ * Therefore, inverse implementers only have to specialize Assignment<Dst,Inverse<...>, ...> for
+ * there own nested expression.
+ *
+ * \sa class Inverse
+ */
+template <typename ArgType>
+struct unary_evaluator<Inverse<ArgType> > : public evaluator<typename Inverse<ArgType>::PlainObject> {
+  typedef Inverse<ArgType> InverseType;
+  typedef typename InverseType::PlainObject PlainObject;
+  typedef evaluator<PlainObject> Base;
+
+  enum { Flags = Base::Flags | EvalBeforeNestingBit };
+
+  EIGEN_DEVICE_FUNC unary_evaluator(const InverseType& inv_xpr) : m_result(inv_xpr.rows(), inv_xpr.cols()) {
+    internal::construct_at<Base>(this, m_result);
+    internal::call_assignment_no_alias(m_result, inv_xpr);
+  }
+
+ protected:
+  PlainObject m_result;
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_INVERSE_H

include/Eigen/src/Core/Map.h

@@ -0,0 +1,153 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2007-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_MAP_H
+#define EIGEN_MAP_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename PlainObjectType, int MapOptions, typename StrideType>
+struct traits<Map<PlainObjectType, MapOptions, StrideType> > : public traits<PlainObjectType> {
+  typedef traits<PlainObjectType> TraitsBase;
+  enum {
+    PlainObjectTypeInnerSize = ((traits<PlainObjectType>::Flags & RowMajorBit) == RowMajorBit)
+                                   ? PlainObjectType::ColsAtCompileTime
+                                   : PlainObjectType::RowsAtCompileTime,
+
+    InnerStrideAtCompileTime = StrideType::InnerStrideAtCompileTime == 0
+                                   ? int(PlainObjectType::InnerStrideAtCompileTime)
+                                   : int(StrideType::InnerStrideAtCompileTime),
+    OuterStrideAtCompileTime = StrideType::OuterStrideAtCompileTime == 0
+                                   ? (InnerStrideAtCompileTime == Dynamic || PlainObjectTypeInnerSize == Dynamic
+                                          ? Dynamic
+                                          : int(InnerStrideAtCompileTime) * int(PlainObjectTypeInnerSize))
+                                   : int(StrideType::OuterStrideAtCompileTime),
+    Alignment = int(MapOptions) & int(AlignedMask),
+    Flags0 = TraitsBase::Flags & (~NestByRefBit),
+    Flags = is_lvalue<PlainObjectType>::value ? int(Flags0) : (int(Flags0) & ~LvalueBit)
+  };
+
+ private:
+  enum { Options };  // Expressions don't have Options
+};
+}  // namespace internal
+
+/** \class Map
+ * \ingroup Core_Module
+ *
+ * \brief A matrix or vector expression mapping an existing array of data.
+ *
+ * \tparam PlainObjectType the equivalent matrix type of the mapped data
+ * \tparam MapOptions specifies the pointer alignment in bytes. It can be: \c #Aligned128, \c #Aligned64, \c #Aligned32,
+ * \c #Aligned16, \c #Aligned8 or \c #Unaligned. The default is \c #Unaligned. \tparam StrideType optionally specifies
+ * strides. By default, Map assumes the memory layout of an ordinary, contiguous array. This can be overridden by
+ * specifying strides. The type passed here must be a specialization of the Stride template, see examples below.
+ *
+ * This class represents a matrix or vector expression mapping an existing array of data.
+ * It can be used to let Eigen interface without any overhead with non-Eigen data structures,
+ * such as plain C arrays or structures from other libraries. By default, it assumes that the
+ * data is laid out contiguously in memory. You can however override this by explicitly specifying
+ * inner and outer strides.
+ *
+ * Here's an example of simply mapping a contiguous array as a \ref TopicStorageOrders "column-major" matrix:
+ * \include Map_simple.cpp
+ * Output: \verbinclude Map_simple.out
+ *
+ * If you need to map non-contiguous arrays, you can do so by specifying strides:
+ *
+ * Here's an example of mapping an array as a vector, specifying an inner stride, that is, the pointer
+ * increment between two consecutive coefficients. Here, we're specifying the inner stride as a compile-time
+ * fixed value.
+ * \include Map_inner_stride.cpp
+ * Output: \verbinclude Map_inner_stride.out
+ *
+ * Here's an example of mapping an array while specifying an outer stride. Here, since we're mapping
+ * as a column-major matrix, 'outer stride' means the pointer increment between two consecutive columns.
+ * Here, we're specifying the outer stride as a runtime parameter. Note that here \c OuterStride<> is
+ * a short version of \c OuterStride<Dynamic> because the default template parameter of OuterStride
+ * is  \c Dynamic
+ * \include Map_outer_stride.cpp
+ * Output: \verbinclude Map_outer_stride.out
+ *
+ * For more details and for an example of specifying both an inner and an outer stride, see class Stride.
+ *
+ * \b Tip: to change the array of data mapped by a Map object, you can use the C++
+ * placement new syntax:
+ *
+ * Example: \include Map_placement_new.cpp
+ * Output: \verbinclude Map_placement_new.out
+ *
+ * This class is the return type of PlainObjectBase::Map() but can also be used directly.
+ *
+ * \sa PlainObjectBase::Map(), \ref TopicStorageOrders
+ */
+template <typename PlainObjectType, int MapOptions, typename StrideType>
+class Map : public MapBase<Map<PlainObjectType, MapOptions, StrideType> > {
+ public:
+  typedef MapBase<Map> Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Map)
+
+  typedef typename Base::PointerType PointerType;
+  typedef PointerType PointerArgType;
+  EIGEN_DEVICE_FUNC inline PointerType cast_to_pointer_type(PointerArgType ptr) { return ptr; }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const {
+    return StrideType::InnerStrideAtCompileTime != 0 ? m_stride.inner() : 1;
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const {
+    return StrideType::OuterStrideAtCompileTime != 0 ? m_stride.outer()
+           : internal::traits<Map>::OuterStrideAtCompileTime != Dynamic
+               ? Index(internal::traits<Map>::OuterStrideAtCompileTime)
+           : IsVectorAtCompileTime    ? (this->size() * innerStride())
+           : int(Flags) & RowMajorBit ? (this->cols() * innerStride())
+                                      : (this->rows() * innerStride());
+  }
+
+  /** Constructor in the fixed-size case.
+   *
+   * \param dataPtr pointer to the array to map
+   * \param stride optional Stride object, passing the strides.
+   */
+  EIGEN_DEVICE_FUNC explicit inline Map(PointerArgType dataPtr, const StrideType& stride = StrideType())
+      : Base(cast_to_pointer_type(dataPtr)), m_stride(stride) {}
+
+  /** Constructor in the dynamic-size vector case.
+   *
+   * \param dataPtr pointer to the array to map
+   * \param size the size of the vector expression
+   * \param stride optional Stride object, passing the strides.
+   */
+  EIGEN_DEVICE_FUNC inline Map(PointerArgType dataPtr, Index size, const StrideType& stride = StrideType())
+      : Base(cast_to_pointer_type(dataPtr), size), m_stride(stride) {}
+
+  /** Constructor in the dynamic-size matrix case.
+   *
+   * \param dataPtr pointer to the array to map
+   * \param rows the number of rows of the matrix expression
+   * \param cols the number of columns of the matrix expression
+   * \param stride optional Stride object, passing the strides.
+   */
+  EIGEN_DEVICE_FUNC inline Map(PointerArgType dataPtr, Index rows, Index cols, const StrideType& stride = StrideType())
+      : Base(cast_to_pointer_type(dataPtr), rows, cols), m_stride(stride) {}
+
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Map)
+
+ protected:
+  StrideType m_stride;
+};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_MAP_H

include/Eigen/src/Core/MapBase.h

@@ -0,0 +1,283 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2007-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_MAPBASE_H
+#define EIGEN_MAPBASE_H
+
+#define EIGEN_STATIC_ASSERT_INDEX_BASED_ACCESS(Derived)                                                               \
+  EIGEN_STATIC_ASSERT((int(internal::evaluator<Derived>::Flags) & LinearAccessBit) || Derived::IsVectorAtCompileTime, \
+                      YOU_ARE_TRYING_TO_USE_AN_INDEX_BASED_ACCESSOR_ON_AN_EXPRESSION_THAT_DOES_NOT_SUPPORT_THAT)
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \ingroup Core_Module
+ *
+ * \brief Base class for dense Map and Block expression with direct access
+ *
+ * This base class provides the const low-level accessors (e.g. coeff, coeffRef) of dense
+ * Map and Block objects with direct access.
+ * Typical users do not have to directly deal with this class.
+ *
+ * This class can be extended by through the macro plugin \c EIGEN_MAPBASE_PLUGIN.
+ * See \link TopicCustomizing_Plugins customizing Eigen \endlink for details.
+ *
+ * The \c Derived class has to provide the following two methods describing the memory layout:
+ *  \code Index innerStride() const; \endcode
+ *  \code Index outerStride() const; \endcode
+ *
+ * \sa class Map, class Block
+ */
+template <typename Derived>
+class MapBase<Derived, ReadOnlyAccessors> : public internal::dense_xpr_base<Derived>::type {
+ public:
+  typedef typename internal::dense_xpr_base<Derived>::type Base;
+  enum {
+    RowsAtCompileTime = internal::traits<Derived>::RowsAtCompileTime,
+    ColsAtCompileTime = internal::traits<Derived>::ColsAtCompileTime,
+    InnerStrideAtCompileTime = internal::traits<Derived>::InnerStrideAtCompileTime,
+    SizeAtCompileTime = Base::SizeAtCompileTime
+  };
+
+  typedef typename internal::traits<Derived>::StorageKind StorageKind;
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+  typedef typename internal::packet_traits<Scalar>::type PacketScalar;
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+  typedef std::conditional_t<bool(internal::is_lvalue<Derived>::value), Scalar*, const Scalar*> PointerType;
+
+  using Base::derived;
+  //    using Base::RowsAtCompileTime;
+  //    using Base::ColsAtCompileTime;
+  //    using Base::SizeAtCompileTime;
+  using Base::Flags;
+  using Base::IsRowMajor;
+  using Base::IsVectorAtCompileTime;
+  using Base::MaxColsAtCompileTime;
+  using Base::MaxRowsAtCompileTime;
+  using Base::MaxSizeAtCompileTime;
+
+  using Base::coeff;
+  using Base::coeffRef;
+  using Base::cols;
+  using Base::eval;
+  using Base::lazyAssign;
+  using Base::rows;
+  using Base::size;
+
+  using Base::colStride;
+  using Base::innerStride;
+  using Base::outerStride;
+  using Base::rowStride;
+
+  // bug 217 - compile error on ICC 11.1
+  using Base::operator=;
+
+  typedef typename Base::CoeffReturnType CoeffReturnType;
+
+  /** \copydoc DenseBase::rows() */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_rows.value(); }
+  /** \copydoc DenseBase::cols() */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_cols.value(); }
+
+  /** Returns a pointer to the first coefficient of the matrix or vector.
+   *
+   * \note When addressing this data, make sure to honor the strides returned by innerStride() and outerStride().
+   *
+   * \sa innerStride(), outerStride()
+   */
+  EIGEN_DEVICE_FUNC constexpr const Scalar* data() const { return m_data; }
+
+  /** \copydoc PlainObjectBase::coeff(Index,Index) const */
+  EIGEN_DEVICE_FUNC inline const Scalar& coeff(Index rowId, Index colId) const {
+    return m_data[colId * colStride() + rowId * rowStride()];
+  }
+
+  /** \copydoc PlainObjectBase::coeff(Index) const */
+  EIGEN_DEVICE_FUNC inline const Scalar& coeff(Index index) const {
+    EIGEN_STATIC_ASSERT_INDEX_BASED_ACCESS(Derived)
+    return m_data[index * innerStride()];
+  }
+
+  /** \copydoc PlainObjectBase::coeffRef(Index,Index) const */
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const {
+    return this->m_data[colId * colStride() + rowId * rowStride()];
+  }
+
+  /** \copydoc PlainObjectBase::coeffRef(Index) const */
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const {
+    EIGEN_STATIC_ASSERT_INDEX_BASED_ACCESS(Derived)
+    return this->m_data[index * innerStride()];
+  }
+
+  /** \internal */
+  template <int LoadMode>
+  inline PacketScalar packet(Index rowId, Index colId) const {
+    return internal::ploadt<PacketScalar, LoadMode>(m_data + (colId * colStride() + rowId * rowStride()));
+  }
+
+  /** \internal */
+  template <int LoadMode>
+  inline PacketScalar packet(Index index) const {
+    EIGEN_STATIC_ASSERT_INDEX_BASED_ACCESS(Derived)
+    return internal::ploadt<PacketScalar, LoadMode>(m_data + index * innerStride());
+  }
+
+  /** \internal Constructor for fixed size matrices or vectors */
+  EIGEN_DEVICE_FUNC explicit inline MapBase(PointerType dataPtr)
+      : m_data(dataPtr), m_rows(RowsAtCompileTime), m_cols(ColsAtCompileTime) {
+    EIGEN_STATIC_ASSERT_FIXED_SIZE(Derived)
+    checkSanity<Derived>();
+  }
+
+  /** \internal Constructor for dynamically sized vectors */
+  EIGEN_DEVICE_FUNC inline MapBase(PointerType dataPtr, Index vecSize)
+      : m_data(dataPtr),
+        m_rows(RowsAtCompileTime == Dynamic ? vecSize : Index(RowsAtCompileTime)),
+        m_cols(ColsAtCompileTime == Dynamic ? vecSize : Index(ColsAtCompileTime)) {
+    EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+    eigen_assert(vecSize >= 0);
+    eigen_assert(dataPtr == 0 || SizeAtCompileTime == Dynamic || SizeAtCompileTime == vecSize);
+    checkSanity<Derived>();
+  }
+
+  /** \internal Constructor for dynamically sized matrices */
+  EIGEN_DEVICE_FUNC inline MapBase(PointerType dataPtr, Index rows, Index cols)
+      : m_data(dataPtr), m_rows(rows), m_cols(cols) {
+    eigen_assert((dataPtr == 0) || (rows >= 0 && (RowsAtCompileTime == Dynamic || RowsAtCompileTime == rows) &&
+                                    cols >= 0 && (ColsAtCompileTime == Dynamic || ColsAtCompileTime == cols)));
+    checkSanity<Derived>();
+  }
+
+#ifdef EIGEN_MAPBASE_PLUGIN
+#include EIGEN_MAPBASE_PLUGIN
+#endif
+
+ protected:
+  EIGEN_DEFAULT_COPY_CONSTRUCTOR(MapBase)
+  EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(MapBase)
+
+  template <typename T>
+  EIGEN_DEVICE_FUNC void checkSanity(std::enable_if_t<(internal::traits<T>::Alignment > 0), void*> = 0) const {
+// Temporary macro to allow scalars to not be properly aligned.  This is while we sort out failures
+// in TensorFlow Lite that are currently relying on this UB.
+#ifndef EIGEN_ALLOW_UNALIGNED_SCALARS
+    // Pointer must be aligned to the Scalar type, otherwise we get UB.
+    eigen_assert((std::uintptr_t(m_data) % alignof(Scalar) == 0) && "data is not scalar-aligned");
+#endif
+#if EIGEN_MAX_ALIGN_BYTES > 0
+    // innerStride() is not set yet when this function is called, so we optimistically assume the lowest plausible
+    // value:
+    const Index minInnerStride = InnerStrideAtCompileTime == Dynamic ? 1 : Index(InnerStrideAtCompileTime);
+    EIGEN_ONLY_USED_FOR_DEBUG(minInnerStride);
+    eigen_assert((((std::uintptr_t(m_data) % internal::traits<Derived>::Alignment) == 0) ||
+                  (cols() * rows() * minInnerStride * sizeof(Scalar)) < internal::traits<Derived>::Alignment) &&
+                 "data is not aligned");
+#endif
+  }
+
+  template <typename T>
+  EIGEN_DEVICE_FUNC void checkSanity(std::enable_if_t<internal::traits<T>::Alignment == 0, void*> = 0) const {
+#ifndef EIGEN_ALLOW_UNALIGNED_SCALARS
+    // Pointer must be aligned to the Scalar type, otherwise we get UB.
+    eigen_assert((std::uintptr_t(m_data) % alignof(Scalar) == 0) && "data is not scalar-aligned");
+#endif
+  }
+
+  PointerType m_data;
+  const internal::variable_if_dynamic<Index, RowsAtCompileTime> m_rows;
+  const internal::variable_if_dynamic<Index, ColsAtCompileTime> m_cols;
+};
+
+/** \ingroup Core_Module
+ *
+ * \brief Base class for non-const dense Map and Block expression with direct access
+ *
+ * This base class provides the non-const low-level accessors (e.g. coeff and coeffRef) of
+ * dense Map and Block objects with direct access.
+ * It inherits MapBase<Derived, ReadOnlyAccessors> which defines the const variant for reading specific entries.
+ *
+ * \sa class Map, class Block
+ */
+template <typename Derived>
+class MapBase<Derived, WriteAccessors> : public MapBase<Derived, ReadOnlyAccessors> {
+  typedef MapBase<Derived, ReadOnlyAccessors> ReadOnlyMapBase;
+
+ public:
+  typedef MapBase<Derived, ReadOnlyAccessors> Base;
+
+  typedef typename Base::Scalar Scalar;
+  typedef typename Base::PacketScalar PacketScalar;
+  typedef typename Base::StorageIndex StorageIndex;
+  typedef typename Base::PointerType PointerType;
+
+  using Base::coeff;
+  using Base::coeffRef;
+  using Base::cols;
+  using Base::derived;
+  using Base::rows;
+  using Base::size;
+
+  using Base::colStride;
+  using Base::innerStride;
+  using Base::outerStride;
+  using Base::rowStride;
+
+  typedef std::conditional_t<internal::is_lvalue<Derived>::value, Scalar, const Scalar> ScalarWithConstIfNotLvalue;
+
+  EIGEN_DEVICE_FUNC constexpr const Scalar* data() const { return this->m_data; }
+  EIGEN_DEVICE_FUNC constexpr ScalarWithConstIfNotLvalue* data() {
+    return this->m_data;
+  }  // no const-cast here so non-const-correct code will give a compile error
+
+  EIGEN_DEVICE_FUNC inline ScalarWithConstIfNotLvalue& coeffRef(Index row, Index col) {
+    return this->m_data[col * colStride() + row * rowStride()];
+  }
+
+  EIGEN_DEVICE_FUNC inline ScalarWithConstIfNotLvalue& coeffRef(Index index) {
+    EIGEN_STATIC_ASSERT_INDEX_BASED_ACCESS(Derived)
+    return this->m_data[index * innerStride()];
+  }
+
+  template <int StoreMode>
+  inline void writePacket(Index row, Index col, const PacketScalar& val) {
+    internal::pstoret<Scalar, PacketScalar, StoreMode>(this->m_data + (col * colStride() + row * rowStride()), val);
+  }
+
+  template <int StoreMode>
+  inline void writePacket(Index index, const PacketScalar& val) {
+    EIGEN_STATIC_ASSERT_INDEX_BASED_ACCESS(Derived)
+    internal::pstoret<Scalar, PacketScalar, StoreMode>(this->m_data + index * innerStride(), val);
+  }
+
+  EIGEN_DEVICE_FUNC explicit inline MapBase(PointerType dataPtr) : Base(dataPtr) {}
+  EIGEN_DEVICE_FUNC inline MapBase(PointerType dataPtr, Index vecSize) : Base(dataPtr, vecSize) {}
+  EIGEN_DEVICE_FUNC inline MapBase(PointerType dataPtr, Index rows, Index cols) : Base(dataPtr, rows, cols) {}
+
+  EIGEN_DEVICE_FUNC Derived& operator=(const MapBase& other) {
+    ReadOnlyMapBase::Base::operator=(other);
+    return derived();
+  }
+
+  // In theory we could simply refer to Base:Base::operator=, but MSVC does not like Base::Base,
+  // see bugs 821 and 920.
+  using ReadOnlyMapBase::Base::operator=;
+
+ protected:
+  EIGEN_DEFAULT_COPY_CONSTRUCTOR(MapBase)
+  EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(MapBase)
+};
+
+#undef EIGEN_STATIC_ASSERT_INDEX_BASED_ACCESS
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_MAPBASE_H

include/Eigen/src/Core/MathFunctionsImpl.h

@@ -0,0 +1,262 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com)
+// Copyright (C) 2016 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_MATHFUNCTIONSIMPL_H
+#define EIGEN_MATHFUNCTIONSIMPL_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+/** \internal Fast reciprocal using Newton-Raphson's method.
+
+ Preconditions:
+   1. The starting guess provided in approx_a_recip must have at least half
+      the leading mantissa bits in the correct result, such that a single
+      Newton-Raphson step is sufficient to get within 1-2 ulps of the correct
+      result.
+   2. If a is zero, approx_a_recip must be infinite with the same sign as a.
+   3. If a is infinite, approx_a_recip must be zero with the same sign as a.
+
+   If the preconditions are satisfied, which they are for for the _*_rcp_ps
+   instructions on x86, the result has a maximum relative error of 2 ulps,
+   and correctly handles reciprocals of zero, infinity, and NaN.
+*/
+template <typename Packet, int Steps>
+struct generic_reciprocal_newton_step {
+  static_assert(Steps > 0, "Steps must be at least 1.");
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet run(const Packet& a, const Packet& approx_a_recip) {
+    using Scalar = typename unpacket_traits<Packet>::type;
+    const Packet two = pset1<Packet>(Scalar(2));
+    // Refine the approximation using one Newton-Raphson step:
+    //   x_{i} = x_{i-1} * (2 - a * x_{i-1})
+    const Packet x = generic_reciprocal_newton_step<Packet, Steps - 1>::run(a, approx_a_recip);
+    const Packet tmp = pnmadd(a, x, two);
+    // If tmp is NaN, it means that a is either +/-0 or +/-Inf.
+    // In this case return the approximation directly.
+    const Packet is_not_nan = pcmp_eq(tmp, tmp);
+    return pselect(is_not_nan, pmul(x, tmp), x);
+  }
+};
+
+template <typename Packet>
+struct generic_reciprocal_newton_step<Packet, 0> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet run(const Packet& /*unused*/, const Packet& approx_rsqrt) {
+    return approx_rsqrt;
+  }
+};
+
+/** \internal Fast reciprocal sqrt using Newton-Raphson's method.
+
+ Preconditions:
+   1. The starting guess provided in approx_a_recip must have at least half
+      the leading mantissa bits in the correct result, such that a single
+      Newton-Raphson step is sufficient to get within 1-2 ulps of the correct
+      result.
+   2. If a is zero, approx_a_recip must be infinite with the same sign as a.
+   3. If a is infinite, approx_a_recip must be zero with the same sign as a.
+
+   If the preconditions are satisfied, which they are for for the _*_rcp_ps
+   instructions on x86, the result has a maximum relative error of 2 ulps,
+   and correctly handles zero, infinity, and NaN. Positive denormals are
+   treated as zero.
+*/
+template <typename Packet, int Steps>
+struct generic_rsqrt_newton_step {
+  static_assert(Steps > 0, "Steps must be at least 1.");
+  using Scalar = typename unpacket_traits<Packet>::type;
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet run(const Packet& a, const Packet& approx_rsqrt) {
+    constexpr Scalar kMinusHalf = Scalar(-1) / Scalar(2);
+    const Packet cst_minus_half = pset1<Packet>(kMinusHalf);
+    const Packet cst_minus_one = pset1<Packet>(Scalar(-1));
+
+    Packet inv_sqrt = approx_rsqrt;
+    for (int step = 0; step < Steps; ++step) {
+      // Refine the approximation using one Newton-Raphson step:
+      // h_n = (x * inv_sqrt) * inv_sqrt - 1 (so that h_n is nearly 0).
+      // inv_sqrt = inv_sqrt - 0.5 * inv_sqrt * h_n
+      Packet r2 = pmul(a, inv_sqrt);
+      Packet half_r = pmul(inv_sqrt, cst_minus_half);
+      Packet h_n = pmadd(r2, inv_sqrt, cst_minus_one);
+      inv_sqrt = pmadd(half_r, h_n, inv_sqrt);
+    }
+
+    // If x is NaN, then either:
+    // 1) the input is NaN
+    // 2) zero and infinity were multiplied
+    // In either of these cases, return approx_rsqrt
+    return pselect(pisnan(inv_sqrt), approx_rsqrt, inv_sqrt);
+  }
+};
+
+template <typename Packet>
+struct generic_rsqrt_newton_step<Packet, 0> {
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet run(const Packet& /*unused*/, const Packet& approx_rsqrt) {
+    return approx_rsqrt;
+  }
+};
+
+/** \internal Fast sqrt using Newton-Raphson's method.
+
+ Preconditions:
+   1. The starting guess for the reciprocal sqrt provided in approx_rsqrt must
+      have at least half the leading mantissa bits in the correct result, such
+      that a single Newton-Raphson step is sufficient to get within 1-2 ulps of
+      the correct result.
+   2. If a is zero, approx_rsqrt must be infinite.
+   3. If a is infinite, approx_rsqrt must be zero.
+
+   If the preconditions are satisfied, which they are for for the _*_rsqrt_ps
+   instructions on x86, the result has a maximum relative error of 2 ulps,
+   and correctly handles zero and infinity, and NaN. Positive denormal inputs
+   are treated as zero.
+*/
+template <typename Packet, int Steps = 1>
+struct generic_sqrt_newton_step {
+  static_assert(Steps > 0, "Steps must be at least 1.");
+
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet run(const Packet& a, const Packet& approx_rsqrt) {
+    using Scalar = typename unpacket_traits<Packet>::type;
+    const Packet one_point_five = pset1<Packet>(Scalar(1.5));
+    const Packet minus_half = pset1<Packet>(Scalar(-0.5));
+    // If a is inf or zero, return a directly.
+    const Packet inf_mask = pcmp_eq(a, pset1<Packet>(NumTraits<Scalar>::infinity()));
+    const Packet return_a = por(pcmp_eq(a, pzero(a)), inf_mask);
+    // Do a single step of Newton's iteration for reciprocal square root:
+    //   x_{n+1} = x_n * (1.5 + (-0.5 * x_n) * (a * x_n))).
+    // The Newton's step is computed this way to avoid over/under-flows.
+    Packet rsqrt = pmul(approx_rsqrt, pmadd(pmul(minus_half, approx_rsqrt), pmul(a, approx_rsqrt), one_point_five));
+    for (int step = 1; step < Steps; ++step) {
+      rsqrt = pmul(rsqrt, pmadd(pmul(minus_half, rsqrt), pmul(a, rsqrt), one_point_five));
+    }
+
+    // Return sqrt(x) = x * rsqrt(x) for non-zero finite positive arguments.
+    // Return a itself for 0 or +inf, NaN for negative arguments.
+    return pselect(return_a, a, pmul(a, rsqrt));
+  }
+};
+
+template <typename RealScalar>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE RealScalar positive_real_hypot(const RealScalar& x, const RealScalar& y) {
+  // IEEE IEC 6059 special cases.
+  if ((numext::isinf)(x) || (numext::isinf)(y)) return NumTraits<RealScalar>::infinity();
+  if ((numext::isnan)(x) || (numext::isnan)(y)) return NumTraits<RealScalar>::quiet_NaN();
+
+  EIGEN_USING_STD(sqrt);
+  RealScalar p, qp;
+  p = numext::maxi(x, y);
+  if (numext::is_exactly_zero(p)) return RealScalar(0);
+  qp = numext::mini(y, x) / p;
+  return p * sqrt(RealScalar(1) + qp * qp);
+}
+
+template <typename Scalar>
+struct hypot_impl {
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+  static EIGEN_DEVICE_FUNC inline RealScalar run(const Scalar& x, const Scalar& y) {
+    EIGEN_USING_STD(abs);
+    return positive_real_hypot<RealScalar>(abs(x), abs(y));
+  }
+};
+
+// Generic complex sqrt implementation that correctly handles corner cases
+// according to https://en.cppreference.com/w/cpp/numeric/complex/sqrt
+template <typename T>
+EIGEN_DEVICE_FUNC std::complex<T> complex_sqrt(const std::complex<T>& z) {
+  // Computes the principal sqrt of the input.
+  //
+  // For a complex square root of the number x + i*y. We want to find real
+  // numbers u and v such that
+  //    (u + i*v)^2 = x + i*y  <=>
+  //    u^2 - v^2 + i*2*u*v = x + i*v.
+  // By equating the real and imaginary parts we get:
+  //    u^2 - v^2 = x
+  //    2*u*v = y.
+  //
+  // For x >= 0, this has the numerically stable solution
+  //    u = sqrt(0.5 * (x + sqrt(x^2 + y^2)))
+  //    v = y / (2 * u)
+  // and for x < 0,
+  //    v = sign(y) * sqrt(0.5 * (-x + sqrt(x^2 + y^2)))
+  //    u = y / (2 * v)
+  //
+  // Letting w = sqrt(0.5 * (|x| + |z|)),
+  //   if x == 0: u = w, v = sign(y) * w
+  //   if x > 0:  u = w, v = y / (2 * w)
+  //   if x < 0:  u = |y| / (2 * w), v = sign(y) * w
+
+  const T x = numext::real(z);
+  const T y = numext::imag(z);
+  const T zero = T(0);
+  const T w = numext::sqrt(T(0.5) * (numext::abs(x) + numext::hypot(x, y)));
+
+  return (numext::isinf)(y)           ? std::complex<T>(NumTraits<T>::infinity(), y)
+         : numext::is_exactly_zero(x) ? std::complex<T>(w, y < zero ? -w : w)
+         : x > zero                   ? std::complex<T>(w, y / (2 * w))
+                                      : std::complex<T>(numext::abs(y) / (2 * w), y < zero ? -w : w);
+}
+
+// Generic complex rsqrt implementation.
+template <typename T>
+EIGEN_DEVICE_FUNC std::complex<T> complex_rsqrt(const std::complex<T>& z) {
+  // Computes the principal reciprocal sqrt of the input.
+  //
+  // For a complex reciprocal square root of the number z = x + i*y. We want to
+  // find real numbers u and v such that
+  //    (u + i*v)^2 = 1 / (x + i*y)  <=>
+  //    u^2 - v^2 + i*2*u*v = x/|z|^2 - i*v/|z|^2.
+  // By equating the real and imaginary parts we get:
+  //    u^2 - v^2 = x/|z|^2
+  //    2*u*v = y/|z|^2.
+  //
+  // For x >= 0, this has the numerically stable solution
+  //    u = sqrt(0.5 * (x + |z|)) / |z|
+  //    v = -y / (2 * u * |z|)
+  // and for x < 0,
+  //    v = -sign(y) * sqrt(0.5 * (-x + |z|)) / |z|
+  //    u = -y / (2 * v * |z|)
+  //
+  // Letting w = sqrt(0.5 * (|x| + |z|)),
+  //   if x == 0: u = w / |z|, v = -sign(y) * w / |z|
+  //   if x > 0:  u = w / |z|, v = -y / (2 * w * |z|)
+  //   if x < 0:  u = |y| / (2 * w * |z|), v = -sign(y) * w / |z|
+
+  const T x = numext::real(z);
+  const T y = numext::imag(z);
+  const T zero = T(0);
+
+  const T abs_z = numext::hypot(x, y);
+  const T w = numext::sqrt(T(0.5) * (numext::abs(x) + abs_z));
+  const T woz = w / abs_z;
+  // Corner cases consistent with 1/sqrt(z) on gcc/clang.
+  return numext::is_exactly_zero(abs_z) ? std::complex<T>(NumTraits<T>::infinity(), NumTraits<T>::quiet_NaN())
+         : ((numext::isinf)(x) || (numext::isinf)(y)) ? std::complex<T>(zero, zero)
+         : numext::is_exactly_zero(x)                 ? std::complex<T>(woz, y < zero ? woz : -woz)
+         : x > zero                                   ? std::complex<T>(woz, -y / (2 * w * abs_z))
+                    : std::complex<T>(numext::abs(y) / (2 * w * abs_z), y < zero ? woz : -woz);
+}
+
+template <typename T>
+EIGEN_DEVICE_FUNC std::complex<T> complex_log(const std::complex<T>& z) {
+  // Computes complex log.
+  T a = numext::abs(z);
+  EIGEN_USING_STD(atan2);
+  T b = atan2(z.imag(), z.real());
+  return std::complex<T>(numext::log(a), b);
+}
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_MATHFUNCTIONSIMPL_H

include/Eigen/src/Core/Matrix.h

@@ -0,0 +1,527 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_MATRIX_H
+#define EIGEN_MATRIX_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename Scalar_, int Rows_, int Cols_, int Options_, int MaxRows_, int MaxCols_>
+struct traits<Matrix<Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_>> {
+ private:
+  constexpr static int size = internal::size_at_compile_time(Rows_, Cols_);
+  typedef typename find_best_packet<Scalar_, size>::type PacketScalar;
+  enum {
+    row_major_bit = Options_ & RowMajor ? RowMajorBit : 0,
+    is_dynamic_size_storage = MaxRows_ == Dynamic || MaxCols_ == Dynamic,
+    max_size = is_dynamic_size_storage ? Dynamic : MaxRows_ * MaxCols_,
+    default_alignment = compute_default_alignment<Scalar_, max_size>::value,
+    actual_alignment = ((Options_ & DontAlign) == 0) ? default_alignment : 0,
+    required_alignment = unpacket_traits<PacketScalar>::alignment,
+    packet_access_bit = (packet_traits<Scalar_>::Vectorizable &&
+                         (EIGEN_UNALIGNED_VECTORIZE || (int(actual_alignment) >= int(required_alignment))))
+                            ? PacketAccessBit
+                            : 0
+  };
+
+ public:
+  typedef Scalar_ Scalar;
+  typedef Dense StorageKind;
+  typedef Eigen::Index StorageIndex;
+  typedef MatrixXpr XprKind;
+  enum {
+    RowsAtCompileTime = Rows_,
+    ColsAtCompileTime = Cols_,
+    MaxRowsAtCompileTime = MaxRows_,
+    MaxColsAtCompileTime = MaxCols_,
+    Flags = compute_matrix_flags(Options_),
+    Options = Options_,
+    InnerStrideAtCompileTime = 1,
+    OuterStrideAtCompileTime = (int(Options) & int(RowMajor)) ? ColsAtCompileTime : RowsAtCompileTime,
+
+    // FIXME, the following flag in only used to define NeedsToAlign in PlainObjectBase
+    EvaluatorFlags = LinearAccessBit | DirectAccessBit | packet_access_bit | row_major_bit,
+    Alignment = actual_alignment
+  };
+};
+}  // namespace internal
+
+/** \class Matrix
+ * \ingroup Core_Module
+ *
+ * \brief The matrix class, also used for vectors and row-vectors
+ *
+ * The %Matrix class is the work-horse for all \em dense (\ref dense "note") matrices and vectors within Eigen.
+ * Vectors are matrices with one column, and row-vectors are matrices with one row.
+ *
+ * The %Matrix class encompasses \em both fixed-size and dynamic-size objects (\ref fixedsize "note").
+ *
+ * The first three template parameters are required:
+ * \tparam Scalar_ Numeric type, e.g. float, double, int or std::complex<float>.
+ *                 User defined scalar types are supported as well (see \ref user_defined_scalars "here").
+ * \tparam Rows_ Number of rows, or \b Dynamic
+ * \tparam Cols_ Number of columns, or \b Dynamic
+ *
+ * The remaining template parameters are optional -- in most cases you don't have to worry about them.
+ * \tparam Options_ A combination of either \b #RowMajor or \b #ColMajor, and of either
+ *                 \b #AutoAlign or \b #DontAlign.
+ *                 The former controls \ref TopicStorageOrders "storage order", and defaults to column-major. The latter
+ * controls alignment, which is required for vectorization. It defaults to aligning matrices except for fixed sizes that
+ * aren't a multiple of the packet size. \tparam MaxRows_ Maximum number of rows. Defaults to \a Rows_ (\ref maxrows
+ * "note"). \tparam MaxCols_ Maximum number of columns. Defaults to \a Cols_ (\ref maxrows "note").
+ *
+ * Eigen provides a number of typedefs covering the usual cases. Here are some examples:
+ *
+ * \li \c Matrix2d is a 2x2 square matrix of doubles (\c Matrix<double, 2, 2>)
+ * \li \c Vector4f is a vector of 4 floats (\c Matrix<float, 4, 1>)
+ * \li \c RowVector3i is a row-vector of 3 ints (\c Matrix<int, 1, 3>)
+ *
+ * \li \c MatrixXf is a dynamic-size matrix of floats (\c Matrix<float, Dynamic, Dynamic>)
+ * \li \c VectorXf is a dynamic-size vector of floats (\c Matrix<float, Dynamic, 1>)
+ *
+ * \li \c Matrix2Xf is a partially fixed-size (dynamic-size) matrix of floats (\c Matrix<float, 2, Dynamic>)
+ * \li \c MatrixX3d is a partially dynamic-size (fixed-size) matrix of double (\c Matrix<double, Dynamic, 3>)
+ *
+ * See \link matrixtypedefs this page \endlink for a complete list of predefined \em %Matrix and \em Vector typedefs.
+ *
+ * You can access elements of vectors and matrices using normal subscripting:
+ *
+ * \code
+ * Eigen::VectorXd v(10);
+ * v[0] = 0.1;
+ * v[1] = 0.2;
+ * v(0) = 0.3;
+ * v(1) = 0.4;
+ *
+ * Eigen::MatrixXi m(10, 10);
+ * m(0, 1) = 1;
+ * m(0, 2) = 2;
+ * m(0, 3) = 3;
+ * \endcode
+ *
+ * This class can be extended with the help of the plugin mechanism described on the page
+ * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_MATRIX_PLUGIN.
+ *
+ * <i><b>Some notes:</b></i>
+ *
+ * <dl>
+ * <dt><b>\anchor dense Dense versus sparse:</b></dt>
+ * <dd>This %Matrix class handles dense, not sparse matrices and vectors. For sparse matrices and vectors, see the
+ * Sparse module.
+ *
+ * Dense matrices and vectors are plain usual arrays of coefficients. All the coefficients are stored, in an ordinary
+ * contiguous array. This is unlike Sparse matrices and vectors where the coefficients are stored as a list of nonzero
+ * coefficients.</dd>
+ *
+ * <dt><b>\anchor fixedsize Fixed-size versus dynamic-size:</b></dt>
+ * <dd>Fixed-size means that the numbers of rows and columns are known at compile-time. In this case, Eigen allocates
+ * the array of coefficients as a fixed-size array, as a class member. This makes sense for very small matrices,
+ * typically up to 4x4, sometimes up to 16x16. Larger matrices should be declared as dynamic-size even if one happens to
+ * know their size at compile-time.
+ *
+ * Dynamic-size means that the numbers of rows or columns are not necessarily known at compile-time. In this case they
+ * are runtime variables, and the array of coefficients is allocated dynamically on the heap.
+ *
+ * Note that \em dense matrices, be they Fixed-size or Dynamic-size, <em>do not</em> expand dynamically in the sense of
+ * a std::map. If you want this behavior, see the Sparse module.</dd>
+ *
+ * <dt><b>\anchor maxrows MaxRows_ and MaxCols_:</b></dt>
+ * <dd>In most cases, one just leaves these parameters to the default values.
+ * These parameters mean the maximum size of rows and columns that the matrix may have. They are useful in cases
+ * when the exact numbers of rows and columns are not known at compile-time, but it is known at compile-time that they
+ * cannot exceed a certain value. This happens when taking dynamic-size blocks inside fixed-size matrices: in this case
+ * MaxRows_ and MaxCols_ are the dimensions of the original matrix, while Rows_ and Cols_ are Dynamic.</dd>
+ * </dl>
+ *
+ * <i><b>ABI and storage layout</b></i>
+ *
+ * The table below summarizes the ABI of some possible Matrix instances which is fixed thorough the lifetime of Eigen 3.
+ * <table  class="manual">
+ * <tr><th>Matrix type</th><th>Equivalent C structure</th></tr>
+ * <tr><td>\code Matrix<T,Dynamic,Dynamic> \endcode</td><td>\code
+ * struct {
+ *   T *data;                  // with (size_t(data)%EIGEN_MAX_ALIGN_BYTES)==0
+ *   Eigen::Index rows, cols;
+ *  };
+ * \endcode</td></tr>
+ * <tr class="alt"><td>\code
+ * Matrix<T,Dynamic,1>
+ * Matrix<T,1,Dynamic> \endcode</td><td>\code
+ * struct {
+ *   T *data;                  // with (size_t(data)%EIGEN_MAX_ALIGN_BYTES)==0
+ *   Eigen::Index size;
+ *  };
+ * \endcode</td></tr>
+ * <tr><td>\code Matrix<T,Rows,Cols> \endcode</td><td>\code
+ * struct {
+ *   T data[Rows*Cols];        // with (size_t(data)%A(Rows*Cols*sizeof(T)))==0
+ *  };
+ * \endcode</td></tr>
+ * <tr class="alt"><td>\code Matrix<T,Dynamic,Dynamic,0,MaxRows,MaxCols> \endcode</td><td>\code
+ * struct {
+ *   T data[MaxRows*MaxCols];  // with (size_t(data)%A(MaxRows*MaxCols*sizeof(T)))==0
+ *   Eigen::Index rows, cols;
+ *  };
+ * \endcode</td></tr>
+ * </table>
+ * Note that in this table Rows, Cols, MaxRows and MaxCols are all positive integers. A(S) is defined to the largest
+ * possible power-of-two smaller to EIGEN_MAX_STATIC_ALIGN_BYTES.
+ *
+ * \see MatrixBase for the majority of the API methods for matrices, \ref TopicClassHierarchy,
+ * \ref TopicStorageOrders
+ */
+
+template <typename Scalar_, int Rows_, int Cols_, int Options_, int MaxRows_, int MaxCols_>
+class Matrix : public PlainObjectBase<Matrix<Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_>> {
+ public:
+  /** \brief Base class typedef.
+   * \sa PlainObjectBase
+   */
+  typedef PlainObjectBase<Matrix> Base;
+
+  enum { Options = Options_ };
+
+  EIGEN_DENSE_PUBLIC_INTERFACE(Matrix)
+
+  typedef typename Base::PlainObject PlainObject;
+
+  using Base::base;
+  using Base::coeffRef;
+
+  /**
+   * \brief Assigns matrices to each other.
+   *
+   * \note This is a special case of the templated operator=. Its purpose is
+   * to prevent a default operator= from hiding the templated operator=.
+   *
+   * \callgraph
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Matrix& operator=(const Matrix& other) { return Base::_set(other); }
+
+  /** \internal
+   * \brief Copies the value of the expression \a other into \c *this with automatic resizing.
+   *
+   * *this might be resized to match the dimensions of \a other. If *this was a null matrix (not already initialized),
+   * it will be initialized.
+   *
+   * Note that copying a row-vector into a vector (and conversely) is allowed.
+   * The resizing, if any, is then done in the appropriate way so that row-vectors
+   * remain row-vectors and vectors remain vectors.
+   */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix& operator=(const DenseBase<OtherDerived>& other) {
+    return Base::_set(other);
+  }
+
+  /* Here, doxygen failed to copy the brief information when using \copydoc */
+
+  /**
+   * \brief Copies the generic expression \a other into *this.
+   * \copydetails DenseBase::operator=(const EigenBase<OtherDerived> &other)
+   */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix& operator=(const EigenBase<OtherDerived>& other) {
+    return Base::operator=(other);
+  }
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix& operator=(const ReturnByValue<OtherDerived>& func) {
+    return Base::operator=(func);
+  }
+
+  /** \brief Default constructor.
+   *
+   * For fixed-size matrices, does nothing.
+   *
+   * For dynamic-size matrices, creates an empty matrix of size 0. Does not allocate any array. Such a matrix
+   * is called a null matrix. This constructor is the unique way to create null matrices: resizing
+   * a matrix to 0 is not supported.
+   *
+   * \sa resize(Index,Index)
+   */
+#if defined(EIGEN_INITIALIZE_COEFFS)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Matrix() { EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED }
+#else
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Matrix() = default;
+#endif
+  /** \brief Move constructor */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Matrix(Matrix&&) = default;
+  /** \brief Moves the matrix into the other one.
+   *
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Matrix& operator=(Matrix&& other)
+      EIGEN_NOEXCEPT_IF(std::is_nothrow_move_assignable<Scalar>::value) {
+    Base::operator=(std::move(other));
+    return *this;
+  }
+
+  /** \copydoc PlainObjectBase(const Scalar&, const Scalar&, const Scalar&,  const Scalar&, const ArgTypes&... args)
+   *
+   * Example: \include Matrix_variadic_ctor_cxx11.cpp
+   * Output: \verbinclude Matrix_variadic_ctor_cxx11.out
+   *
+   * \sa Matrix(const std::initializer_list<std::initializer_list<Scalar>>&)
+   */
+  template <typename... ArgTypes>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Scalar& a0, const Scalar& a1, const Scalar& a2, const Scalar& a3,
+                                               const ArgTypes&... args)
+      : Base(a0, a1, a2, a3, args...) {}
+
+  /** \brief Constructs a Matrix and initializes it from the coefficients given as initializer-lists grouped by row.
+   * \cpp11
+   *
+   * In the general case, the constructor takes a list of rows, each row being represented as a list of coefficients:
+   *
+   * Example: \include Matrix_initializer_list_23_cxx11.cpp
+   * Output: \verbinclude Matrix_initializer_list_23_cxx11.out
+   *
+   * Each of the inner initializer lists must contain the exact same number of elements, otherwise an assertion is
+   * triggered.
+   *
+   * In the case of a compile-time column vector, implicit transposition from a single row is allowed.
+   * Therefore <code>VectorXd{{1,2,3,4,5}}</code> is legal and the more verbose syntax
+   * <code>RowVectorXd{{1},{2},{3},{4},{5}}</code> can be avoided:
+   *
+   * Example: \include Matrix_initializer_list_vector_cxx11.cpp
+   * Output: \verbinclude Matrix_initializer_list_vector_cxx11.out
+   *
+   * In the case of fixed-sized matrices, the initializer list sizes must exactly match the matrix sizes,
+   * and implicit transposition is allowed for compile-time vectors only.
+   *
+   * \sa Matrix(const Scalar& a0, const Scalar& a1, const Scalar& a2,  const Scalar& a3, const ArgTypes&... args)
+   */
+  EIGEN_DEVICE_FUNC explicit constexpr EIGEN_STRONG_INLINE Matrix(
+      const std::initializer_list<std::initializer_list<Scalar>>& list)
+      : Base(list) {}
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+
+  // This constructor is for both 1x1 matrices and dynamic vectors
+  template <typename T>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Matrix(const T& x) {
+    Base::template _init1<T>(x);
+  }
+
+  template <typename T0, typename T1>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const T0& x, const T1& y) {
+    Base::template _init2<T0, T1>(x, y);
+  }
+
+#else
+  /** \brief Constructs a fixed-sized matrix initialized with coefficients starting at \a data */
+  EIGEN_DEVICE_FUNC explicit Matrix(const Scalar* data);
+
+  /** \brief Constructs a vector or row-vector with given dimension. \only_for_vectors
+   *
+   * This is useful for dynamic-size vectors. For fixed-size vectors,
+   * it is redundant to pass these parameters, so one should use the default constructor
+   * Matrix() instead.
+   *
+   * \warning This constructor is disabled for fixed-size \c 1x1 matrices. For instance,
+   * calling Matrix<double,1,1>(1) will call the initialization constructor: Matrix(const Scalar&).
+   * For fixed-size \c 1x1 matrices it is therefore recommended to use the default
+   * constructor Matrix() instead, especially when using one of the non standard
+   * \c EIGEN_INITIALIZE_MATRICES_BY_{ZERO,\c NAN} macros (see \ref TopicPreprocessorDirectives).
+   */
+  EIGEN_STRONG_INLINE explicit Matrix(Index dim);
+  /** \brief Constructs an initialized 1x1 matrix with the given coefficient
+   * \sa Matrix(const Scalar&, const Scalar&, const Scalar&,  const Scalar&, const ArgTypes&...) */
+  Matrix(const Scalar& x);
+  /** \brief Constructs an uninitialized matrix with \a rows rows and \a cols columns.
+   *
+   * This is useful for dynamic-size matrices. For fixed-size matrices,
+   * it is redundant to pass these parameters, so one should use the default constructor
+   * Matrix() instead.
+   *
+   * \warning This constructor is disabled for fixed-size \c 1x2 and \c 2x1 vectors. For instance,
+   * calling Matrix2f(2,1) will call the initialization constructor: Matrix(const Scalar& x, const Scalar& y).
+   * For fixed-size \c 1x2 or \c 2x1 vectors it is therefore recommended to use the default
+   * constructor Matrix() instead, especially when using one of the non standard
+   * \c EIGEN_INITIALIZE_MATRICES_BY_{ZERO,\c NAN} macros (see \ref TopicPreprocessorDirectives).
+   */
+  EIGEN_DEVICE_FUNC Matrix(Index rows, Index cols);
+
+  /** \brief Constructs an initialized 2D vector with given coefficients
+   * \sa Matrix(const Scalar&, const Scalar&, const Scalar&,  const Scalar&, const ArgTypes&...) */
+  Matrix(const Scalar& x, const Scalar& y);
+#endif  // end EIGEN_PARSED_BY_DOXYGEN
+
+  /** \brief Constructs an initialized 3D vector with given coefficients
+   * \sa Matrix(const Scalar&, const Scalar&, const Scalar&,  const Scalar&, const ArgTypes&...)
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Scalar& x, const Scalar& y, const Scalar& z) {
+    EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Matrix, 3)
+    m_storage.data()[0] = x;
+    m_storage.data()[1] = y;
+    m_storage.data()[2] = z;
+  }
+  /** \brief Constructs an initialized 4D vector with given coefficients
+   * \sa Matrix(const Scalar&, const Scalar&, const Scalar&,  const Scalar&, const ArgTypes&...)
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const Scalar& x, const Scalar& y, const Scalar& z, const Scalar& w) {
+    EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Matrix, 4)
+    m_storage.data()[0] = x;
+    m_storage.data()[1] = y;
+    m_storage.data()[2] = z;
+    m_storage.data()[3] = w;
+  }
+
+  /** \brief Copy constructor */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr Matrix(const Matrix&) = default;
+
+  /** \brief Copy constructor for generic expressions.
+   * \sa MatrixBase::operator=(const EigenBase<OtherDerived>&)
+   */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Matrix(const EigenBase<OtherDerived>& other) : Base(other.derived()) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return 1; }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return this->innerSize(); }
+
+  /////////// Geometry module ///////////
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC explicit Matrix(const RotationBase<OtherDerived, ColsAtCompileTime>& r);
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC Matrix& operator=(const RotationBase<OtherDerived, ColsAtCompileTime>& r);
+
+// allow to extend Matrix outside Eigen
+#ifdef EIGEN_MATRIX_PLUGIN
+#include EIGEN_MATRIX_PLUGIN
+#endif
+
+ protected:
+  template <typename Derived, typename OtherDerived, bool IsVector>
+  friend struct internal::conservative_resize_like_impl;
+
+  using Base::m_storage;
+};
+
+/** \defgroup matrixtypedefs Global matrix typedefs
+ *
+ * \ingroup Core_Module
+ *
+ * %Eigen defines several typedef shortcuts for most common matrix and vector types.
+ *
+ * The general patterns are the following:
+ *
+ * \c MatrixSizeType where \c Size can be \c 2,\c 3,\c 4 for fixed size square matrices or \c X for dynamic size,
+ * and where \c Type can be \c i for integer, \c f for float, \c d for double, \c cf for complex float, \c cd
+ * for complex double.
+ *
+ * For example, \c Matrix3d is a fixed-size 3x3 matrix type of doubles, and \c MatrixXf is a dynamic-size matrix of
+ * floats.
+ *
+ * There are also \c VectorSizeType and \c RowVectorSizeType which are self-explanatory. For example, \c Vector4cf is
+ * a fixed-size vector of 4 complex floats.
+ *
+ * With \cpp11, template alias are also defined for common sizes.
+ * They follow the same pattern as above except that the scalar type suffix is replaced by a
+ * template parameter, i.e.:
+ *   - `MatrixSize<Type>` where `Size` can be \c 2,\c 3,\c 4 for fixed size square matrices or \c X for dynamic size.
+ *   - `MatrixXSize<Type>` and `MatrixSizeX<Type>` where `Size` can be \c 2,\c 3,\c 4 for hybrid dynamic/fixed matrices.
+ *   - `VectorSize<Type>` and `RowVectorSize<Type>` for column and row vectors.
+ *
+ * With \cpp11, you can also use fully generic column and row vector types: `Vector<Type,Size>` and
+ * `RowVector<Type,Size>`.
+ *
+ * \sa class Matrix
+ */
+
+#define EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, Size, SizeSuffix)    \
+  /** \ingroup matrixtypedefs */                                   \
+  /** \brief `Size`&times;`Size` matrix of type `Type`. */         \
+  typedef Matrix<Type, Size, Size> Matrix##SizeSuffix##TypeSuffix; \
+  /** \ingroup matrixtypedefs */                                   \
+  /** \brief `Size`&times;`1` vector of type `Type`. */            \
+  typedef Matrix<Type, Size, 1> Vector##SizeSuffix##TypeSuffix;    \
+  /** \ingroup matrixtypedefs */                                   \
+  /** \brief `1`&times;`Size` vector of type `Type`. */            \
+  typedef Matrix<Type, 1, Size> RowVector##SizeSuffix##TypeSuffix;
+
+#define EIGEN_MAKE_FIXED_TYPEDEFS(Type, TypeSuffix, Size)          \
+  /** \ingroup matrixtypedefs */                                   \
+  /** \brief `Size`&times;`Dynamic` matrix of type `Type`. */      \
+  typedef Matrix<Type, Size, Dynamic> Matrix##Size##X##TypeSuffix; \
+  /** \ingroup matrixtypedefs */                                   \
+  /** \brief `Dynamic`&times;`Size` matrix of type `Type`. */      \
+  typedef Matrix<Type, Dynamic, Size> Matrix##X##Size##TypeSuffix;
+
+#define EIGEN_MAKE_TYPEDEFS_ALL_SIZES(Type, TypeSuffix) \
+  EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, 2, 2)           \
+  EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, 3, 3)           \
+  EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, 4, 4)           \
+  EIGEN_MAKE_TYPEDEFS(Type, TypeSuffix, Dynamic, X)     \
+  EIGEN_MAKE_FIXED_TYPEDEFS(Type, TypeSuffix, 2)        \
+  EIGEN_MAKE_FIXED_TYPEDEFS(Type, TypeSuffix, 3)        \
+  EIGEN_MAKE_FIXED_TYPEDEFS(Type, TypeSuffix, 4)
+
+EIGEN_MAKE_TYPEDEFS_ALL_SIZES(int, i)
+EIGEN_MAKE_TYPEDEFS_ALL_SIZES(float, f)
+EIGEN_MAKE_TYPEDEFS_ALL_SIZES(double, d)
+EIGEN_MAKE_TYPEDEFS_ALL_SIZES(std::complex<float>, cf)
+EIGEN_MAKE_TYPEDEFS_ALL_SIZES(std::complex<double>, cd)
+
+#undef EIGEN_MAKE_TYPEDEFS_ALL_SIZES
+#undef EIGEN_MAKE_TYPEDEFS
+#undef EIGEN_MAKE_FIXED_TYPEDEFS
+
+#define EIGEN_MAKE_TYPEDEFS(Size, SizeSuffix)                    \
+  /** \ingroup matrixtypedefs */                                 \
+  /** \brief \cpp11 `Size`&times;`Size` matrix of type `Type`.*/ \
+  template <typename Type>                                       \
+  using Matrix##SizeSuffix = Matrix<Type, Size, Size>;           \
+  /** \ingroup matrixtypedefs */                                 \
+  /** \brief \cpp11 `Size`&times;`1` vector of type `Type`.*/    \
+  template <typename Type>                                       \
+  using Vector##SizeSuffix = Matrix<Type, Size, 1>;              \
+  /** \ingroup matrixtypedefs */                                 \
+  /** \brief \cpp11 `1`&times;`Size` vector of type `Type`.*/    \
+  template <typename Type>                                       \
+  using RowVector##SizeSuffix = Matrix<Type, 1, Size>;
+
+#define EIGEN_MAKE_FIXED_TYPEDEFS(Size)                              \
+  /** \ingroup matrixtypedefs */                                     \
+  /** \brief \cpp11 `Size`&times;`Dynamic` matrix of type `Type` */  \
+  template <typename Type>                                           \
+  using Matrix##Size##X = Matrix<Type, Size, Dynamic>;               \
+  /** \ingroup matrixtypedefs */                                     \
+  /** \brief \cpp11 `Dynamic`&times;`Size` matrix of type `Type`. */ \
+  template <typename Type>                                           \
+  using Matrix##X##Size = Matrix<Type, Dynamic, Size>;
+
+EIGEN_MAKE_TYPEDEFS(2, 2)
+EIGEN_MAKE_TYPEDEFS(3, 3)
+EIGEN_MAKE_TYPEDEFS(4, 4)
+EIGEN_MAKE_TYPEDEFS(Dynamic, X)
+EIGEN_MAKE_FIXED_TYPEDEFS(2)
+EIGEN_MAKE_FIXED_TYPEDEFS(3)
+EIGEN_MAKE_FIXED_TYPEDEFS(4)
+
+/** \ingroup matrixtypedefs
+ * \brief \cpp11 `Size`&times;`1` vector of type `Type`. */
+template <typename Type, int Size>
+using Vector = Matrix<Type, Size, 1>;
+
+/** \ingroup matrixtypedefs
+ * \brief \cpp11 `1`&times;`Size` vector of type `Type`. */
+template <typename Type, int Size>
+using RowVector = Matrix<Type, 1, Size>;
+
+#undef EIGEN_MAKE_TYPEDEFS
+#undef EIGEN_MAKE_FIXED_TYPEDEFS
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_MATRIX_H

include/Eigen/src/Core/MatrixBase.h

@@ -0,0 +1,542 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2009 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_MATRIXBASE_H
+#define EIGEN_MATRIXBASE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class MatrixBase
+  * \ingroup Core_Module
+  *
+  * \brief Base class for all dense matrices, vectors, and expressions
+  *
+  * This class is the base that is inherited by all matrix, vector, and related expression
+  * types. Most of the Eigen API is contained in this class, and its base classes. Other important
+  * classes for the Eigen API are Matrix, and VectorwiseOp.
+  *
+  * Note that some methods are defined in other modules such as the \ref LU_Module LU module
+  * for all functions related to matrix inversions.
+  *
+  * \tparam Derived is the derived type, e.g. a matrix type, or an expression, etc.
+  *
+  * When writing a function taking Eigen objects as argument, if you want your function
+  * to take as argument any matrix, vector, or expression, just let it take a
+  * MatrixBase argument. As an example, here is a function printFirstRow which, given
+  * a matrix, vector, or expression \a x, prints the first row of \a x.
+  *
+  * \code
+    template<typename Derived>
+    void printFirstRow(const Eigen::MatrixBase<Derived>& x)
+    {
+      cout << x.row(0) << endl;
+    }
+  * \endcode
+  *
+  * This class can be extended with the help of the plugin mechanism described on the page
+  * \ref TopicCustomizing_Plugins by defining the preprocessor symbol \c EIGEN_MATRIXBASE_PLUGIN.
+  *
+  * \sa \blank \ref TopicClassHierarchy
+  */
+template <typename Derived>
+class MatrixBase : public DenseBase<Derived> {
+ public:
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef MatrixBase StorageBaseType;
+  typedef typename internal::traits<Derived>::StorageKind StorageKind;
+  typedef typename internal::traits<Derived>::StorageIndex StorageIndex;
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+  typedef typename internal::packet_traits<Scalar>::type PacketScalar;
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+
+  typedef DenseBase<Derived> Base;
+  using Base::ColsAtCompileTime;
+  using Base::Flags;
+  using Base::IsVectorAtCompileTime;
+  using Base::MaxColsAtCompileTime;
+  using Base::MaxRowsAtCompileTime;
+  using Base::MaxSizeAtCompileTime;
+  using Base::RowsAtCompileTime;
+  using Base::SizeAtCompileTime;
+
+  using Base::coeff;
+  using Base::coeffRef;
+  using Base::cols;
+  using Base::const_cast_derived;
+  using Base::derived;
+  using Base::eval;
+  using Base::lazyAssign;
+  using Base::rows;
+  using Base::size;
+  using Base::operator-;
+  using Base::operator+=;
+  using Base::operator-=;
+  using Base::operator*=;
+  using Base::operator/=;
+
+  typedef typename Base::CoeffReturnType CoeffReturnType;
+  typedef typename Base::ConstTransposeReturnType ConstTransposeReturnType;
+  typedef typename Base::RowXpr RowXpr;
+  typedef typename Base::ColXpr ColXpr;
+#endif  // not EIGEN_PARSED_BY_DOXYGEN
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** type of the equivalent square matrix */
+  typedef Matrix<Scalar, internal::max_size_prefer_dynamic(RowsAtCompileTime, ColsAtCompileTime),
+                 internal::max_size_prefer_dynamic(RowsAtCompileTime, ColsAtCompileTime)>
+      SquareMatrixType;
+#endif  // not EIGEN_PARSED_BY_DOXYGEN
+
+  /** \returns the size of the main diagonal, which is min(rows(),cols()).
+   * \sa rows(), cols(), SizeAtCompileTime. */
+  EIGEN_DEVICE_FUNC inline Index diagonalSize() const { return (numext::mini)(rows(), cols()); }
+
+  typedef typename Base::PlainObject PlainObject;
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** \internal Represents a matrix with all coefficients equal to one another*/
+  typedef CwiseNullaryOp<internal::scalar_constant_op<Scalar>, PlainObject> ConstantReturnType;
+  /** \internal the return type of MatrixBase::adjoint() */
+  typedef std::conditional_t<NumTraits<Scalar>::IsComplex,
+                             CwiseUnaryOp<internal::scalar_conjugate_op<Scalar>, ConstTransposeReturnType>,
+                             ConstTransposeReturnType>
+      AdjointReturnType;
+  /** \internal Return type of eigenvalues() */
+  typedef Matrix<std::complex<RealScalar>, internal::traits<Derived>::ColsAtCompileTime, 1, ColMajor>
+      EigenvaluesReturnType;
+  /** \internal the return type of identity */
+  typedef CwiseNullaryOp<internal::scalar_identity_op<Scalar>, PlainObject> IdentityReturnType;
+  /** \internal the return type of unit vectors */
+  typedef Block<const CwiseNullaryOp<internal::scalar_identity_op<Scalar>, SquareMatrixType>,
+                internal::traits<Derived>::RowsAtCompileTime, internal::traits<Derived>::ColsAtCompileTime>
+      BasisReturnType;
+#endif  // not EIGEN_PARSED_BY_DOXYGEN
+
+#define EIGEN_CURRENT_STORAGE_BASE_CLASS Eigen::MatrixBase
+#define EIGEN_DOC_UNARY_ADDONS(X, Y)
+#include "../plugins/CommonCwiseBinaryOps.inc"
+#include "../plugins/MatrixCwiseUnaryOps.inc"
+#include "../plugins/MatrixCwiseBinaryOps.inc"
+#ifdef EIGEN_MATRIXBASE_PLUGIN
+#include EIGEN_MATRIXBASE_PLUGIN
+#endif
+#undef EIGEN_CURRENT_STORAGE_BASE_CLASS
+#undef EIGEN_DOC_UNARY_ADDONS
+
+  /** Special case of the template operator=, in order to prevent the compiler
+   * from generating a default operator= (issue hit with g++ 4.1)
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const MatrixBase& other);
+
+  // We cannot inherit here via Base::operator= since it is causing
+  // trouble with MSVC.
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const DenseBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC Derived& operator=(const EigenBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC Derived& operator=(const ReturnByValue<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator+=(const MatrixBase<OtherDerived>& other);
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator-=(const MatrixBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC const Product<Derived, OtherDerived> operator*(const MatrixBase<OtherDerived>& other) const;
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC const Product<Derived, OtherDerived, LazyProduct> lazyProduct(
+      const MatrixBase<OtherDerived>& other) const;
+
+  template <typename OtherDerived>
+  Derived& operator*=(const EigenBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  void applyOnTheLeft(const EigenBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  void applyOnTheRight(const EigenBase<OtherDerived>& other);
+
+  template <typename DiagonalDerived>
+  EIGEN_DEVICE_FUNC const Product<Derived, DiagonalDerived, LazyProduct> operator*(
+      const DiagonalBase<DiagonalDerived>& diagonal) const;
+
+  template <typename SkewDerived>
+  EIGEN_DEVICE_FUNC const Product<Derived, SkewDerived, LazyProduct> operator*(
+      const SkewSymmetricBase<SkewDerived>& skew) const;
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC typename ScalarBinaryOpTraits<typename internal::traits<Derived>::Scalar,
+                                                  typename internal::traits<OtherDerived>::Scalar>::ReturnType
+  dot(const MatrixBase<OtherDerived>& other) const;
+
+  EIGEN_DEVICE_FUNC RealScalar squaredNorm() const;
+  EIGEN_DEVICE_FUNC RealScalar norm() const;
+  RealScalar stableNorm() const;
+  RealScalar blueNorm() const;
+  RealScalar hypotNorm() const;
+  EIGEN_DEVICE_FUNC const PlainObject normalized() const;
+  EIGEN_DEVICE_FUNC const PlainObject stableNormalized() const;
+  EIGEN_DEVICE_FUNC void normalize();
+  EIGEN_DEVICE_FUNC void stableNormalize();
+
+  EIGEN_DEVICE_FUNC const AdjointReturnType adjoint() const;
+  EIGEN_DEVICE_FUNC void adjointInPlace();
+
+  typedef Diagonal<Derived> DiagonalReturnType;
+  EIGEN_DEVICE_FUNC DiagonalReturnType diagonal();
+
+  typedef Diagonal<const Derived> ConstDiagonalReturnType;
+  EIGEN_DEVICE_FUNC const ConstDiagonalReturnType diagonal() const;
+
+  template <int Index>
+  EIGEN_DEVICE_FUNC Diagonal<Derived, Index> diagonal();
+
+  template <int Index>
+  EIGEN_DEVICE_FUNC const Diagonal<const Derived, Index> diagonal() const;
+
+  EIGEN_DEVICE_FUNC Diagonal<Derived, DynamicIndex> diagonal(Index index);
+  EIGEN_DEVICE_FUNC const Diagonal<const Derived, DynamicIndex> diagonal(Index index) const;
+
+  template <unsigned int Mode>
+  struct TriangularViewReturnType {
+    typedef TriangularView<Derived, Mode> Type;
+  };
+  template <unsigned int Mode>
+  struct ConstTriangularViewReturnType {
+    typedef const TriangularView<const Derived, Mode> Type;
+  };
+
+  template <unsigned int Mode>
+  EIGEN_DEVICE_FUNC typename TriangularViewReturnType<Mode>::Type triangularView();
+  template <unsigned int Mode>
+  EIGEN_DEVICE_FUNC typename ConstTriangularViewReturnType<Mode>::Type triangularView() const;
+
+  template <unsigned int UpLo>
+  struct SelfAdjointViewReturnType {
+    typedef SelfAdjointView<Derived, UpLo> Type;
+  };
+  template <unsigned int UpLo>
+  struct ConstSelfAdjointViewReturnType {
+    typedef const SelfAdjointView<const Derived, UpLo> Type;
+  };
+
+  template <unsigned int UpLo>
+  EIGEN_DEVICE_FUNC typename SelfAdjointViewReturnType<UpLo>::Type selfadjointView();
+  template <unsigned int UpLo>
+  EIGEN_DEVICE_FUNC typename ConstSelfAdjointViewReturnType<UpLo>::Type selfadjointView() const;
+
+  const SparseView<Derived> sparseView(
+      const Scalar& m_reference = Scalar(0),
+      const typename NumTraits<Scalar>::Real& m_epsilon = NumTraits<Scalar>::dummy_precision()) const;
+  EIGEN_DEVICE_FUNC static const IdentityReturnType Identity();
+  EIGEN_DEVICE_FUNC static const IdentityReturnType Identity(Index rows, Index cols);
+  EIGEN_DEVICE_FUNC static const BasisReturnType Unit(Index size, Index i);
+  EIGEN_DEVICE_FUNC static const BasisReturnType Unit(Index i);
+  EIGEN_DEVICE_FUNC static const BasisReturnType UnitX();
+  EIGEN_DEVICE_FUNC static const BasisReturnType UnitY();
+  EIGEN_DEVICE_FUNC static const BasisReturnType UnitZ();
+  EIGEN_DEVICE_FUNC static const BasisReturnType UnitW();
+
+  EIGEN_DEVICE_FUNC const DiagonalWrapper<const Derived> asDiagonal() const;
+  const PermutationWrapper<const Derived> asPermutation() const;
+  EIGEN_DEVICE_FUNC const SkewSymmetricWrapper<const Derived> asSkewSymmetric() const;
+
+  EIGEN_DEVICE_FUNC Derived& setIdentity();
+  EIGEN_DEVICE_FUNC Derived& setIdentity(Index rows, Index cols);
+  EIGEN_DEVICE_FUNC Derived& setUnit(Index i);
+  EIGEN_DEVICE_FUNC Derived& setUnit(Index newSize, Index i);
+
+  bool isIdentity(const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+  bool isDiagonal(const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+
+  bool isUpperTriangular(const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+  bool isLowerTriangular(const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+
+  bool isSkewSymmetric(const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+
+  template <typename OtherDerived>
+  bool isOrthogonal(const MatrixBase<OtherDerived>& other,
+                    const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+  bool isUnitary(const RealScalar& prec = NumTraits<Scalar>::dummy_precision()) const;
+
+  /** \returns true if each coefficients of \c *this and \a other are all exactly equal.
+   * \warning When using floating point scalar values you probably should rather use a
+   *          fuzzy comparison such as isApprox()
+   * \sa isApprox(), operator!= */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline bool operator==(const MatrixBase<OtherDerived>& other) const {
+    return (this->rows() == other.rows()) && (this->cols() == other.cols()) && cwiseEqual(other).all();
+  }
+
+  /** \returns true if at least one pair of coefficients of \c *this and \a other are not exactly equal to each other.
+   * \warning When using floating point scalar values you probably should rather use a
+   *          fuzzy comparison such as isApprox()
+   * \sa isApprox(), operator== */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline bool operator!=(const MatrixBase<OtherDerived>& other) const {
+    return !(*this == other);
+  }
+
+  NoAlias<Derived, Eigen::MatrixBase> EIGEN_DEVICE_FUNC noalias();
+
+  // TODO forceAlignedAccess is temporarily disabled
+  // Need to find a nicer workaround.
+  inline const Derived& forceAlignedAccess() const { return derived(); }
+  inline Derived& forceAlignedAccess() { return derived(); }
+  template <bool Enable>
+  inline const Derived& forceAlignedAccessIf() const {
+    return derived();
+  }
+  template <bool Enable>
+  inline Derived& forceAlignedAccessIf() {
+    return derived();
+  }
+
+  EIGEN_DEVICE_FUNC Scalar trace() const;
+
+  template <int p>
+  EIGEN_DEVICE_FUNC RealScalar lpNorm() const;
+
+  EIGEN_DEVICE_FUNC MatrixBase<Derived>& matrix() { return *this; }
+  EIGEN_DEVICE_FUNC const MatrixBase<Derived>& matrix() const { return *this; }
+
+  /** \returns an \link Eigen::ArrayBase Array \endlink expression of this matrix
+   * \sa ArrayBase::matrix() */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ArrayWrapper<Derived> array() { return ArrayWrapper<Derived>(derived()); }
+  /** \returns a const \link Eigen::ArrayBase Array \endlink expression of this matrix
+   * \sa ArrayBase::matrix() */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const ArrayWrapper<const Derived> array() const {
+    return ArrayWrapper<const Derived>(derived());
+  }
+
+  /////////// LU module ///////////
+
+  template <typename PermutationIndex = DefaultPermutationIndex>
+  inline const FullPivLU<PlainObject, PermutationIndex> fullPivLu() const;
+  template <typename PermutationIndex = DefaultPermutationIndex>
+  inline const PartialPivLU<PlainObject, PermutationIndex> partialPivLu() const;
+
+  template <typename PermutationIndex = DefaultPermutationIndex>
+  inline const PartialPivLU<PlainObject, PermutationIndex> lu() const;
+
+  EIGEN_DEVICE_FUNC inline const Inverse<Derived> inverse() const;
+
+  template <typename ResultType>
+  inline void computeInverseAndDetWithCheck(
+      ResultType& inverse, typename ResultType::Scalar& determinant, bool& invertible,
+      const RealScalar& absDeterminantThreshold = NumTraits<Scalar>::dummy_precision()) const;
+
+  template <typename ResultType>
+  inline void computeInverseWithCheck(
+      ResultType& inverse, bool& invertible,
+      const RealScalar& absDeterminantThreshold = NumTraits<Scalar>::dummy_precision()) const;
+
+  EIGEN_DEVICE_FUNC Scalar determinant() const;
+
+  /////////// Cholesky module ///////////
+
+  inline const LLT<PlainObject> llt() const;
+  inline const LDLT<PlainObject> ldlt() const;
+
+  /////////// QR module ///////////
+
+  inline const HouseholderQR<PlainObject> householderQr() const;
+  template <typename PermutationIndex = DefaultPermutationIndex>
+  inline const ColPivHouseholderQR<PlainObject, PermutationIndex> colPivHouseholderQr() const;
+  template <typename PermutationIndex = DefaultPermutationIndex>
+  inline const FullPivHouseholderQR<PlainObject, PermutationIndex> fullPivHouseholderQr() const;
+  template <typename PermutationIndex = DefaultPermutationIndex>
+  inline const CompleteOrthogonalDecomposition<PlainObject, PermutationIndex> completeOrthogonalDecomposition() const;
+
+  /////////// Eigenvalues module ///////////
+
+  inline EigenvaluesReturnType eigenvalues() const;
+  inline RealScalar operatorNorm() const;
+
+  /////////// SVD module ///////////
+
+  template <int Options = 0>
+  inline JacobiSVD<PlainObject, Options> jacobiSvd() const;
+  template <int Options = 0>
+  EIGEN_DEPRECATED inline JacobiSVD<PlainObject, Options> jacobiSvd(unsigned int computationOptions) const;
+
+  template <int Options = 0>
+  inline BDCSVD<PlainObject, Options> bdcSvd() const;
+  template <int Options = 0>
+  EIGEN_DEPRECATED inline BDCSVD<PlainObject, Options> bdcSvd(unsigned int computationOptions) const;
+
+  /////////// Geometry module ///////////
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline typename internal::cross_impl<Derived, OtherDerived>::return_type cross(
+      const MatrixBase<OtherDerived>& other) const;
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline PlainObject cross3(const MatrixBase<OtherDerived>& other) const;
+
+  EIGEN_DEVICE_FUNC inline PlainObject unitOrthogonal(void) const;
+
+  EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline Matrix<Scalar, 3, 1> eulerAngles(Index a0, Index a1, Index a2) const;
+
+  EIGEN_DEVICE_FUNC inline Matrix<Scalar, 3, 1> canonicalEulerAngles(Index a0, Index a1, Index a2) const;
+
+  // put this as separate enum value to work around possible GCC 4.3 bug (?)
+  enum {
+    HomogeneousReturnTypeDirection =
+        ColsAtCompileTime == 1 && RowsAtCompileTime == 1
+            ? ((internal::traits<Derived>::Flags & RowMajorBit) == RowMajorBit ? Horizontal : Vertical)
+        : ColsAtCompileTime == 1 ? Vertical
+                                 : Horizontal
+  };
+  typedef Homogeneous<Derived, HomogeneousReturnTypeDirection> HomogeneousReturnType;
+  EIGEN_DEVICE_FUNC inline HomogeneousReturnType homogeneous() const;
+
+  enum { SizeMinusOne = SizeAtCompileTime == Dynamic ? Dynamic : SizeAtCompileTime - 1 };
+  typedef Block<const Derived, internal::traits<Derived>::ColsAtCompileTime == 1 ? SizeMinusOne : 1,
+                internal::traits<Derived>::ColsAtCompileTime == 1 ? 1 : SizeMinusOne>
+      ConstStartMinusOne;
+  typedef EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(ConstStartMinusOne, Scalar, quotient) HNormalizedReturnType;
+  EIGEN_DEVICE_FUNC inline const HNormalizedReturnType hnormalized() const;
+
+  ////////// Householder module ///////////
+
+  EIGEN_DEVICE_FUNC void makeHouseholderInPlace(Scalar& tau, RealScalar& beta);
+  template <typename EssentialPart>
+  EIGEN_DEVICE_FUNC void makeHouseholder(EssentialPart& essential, Scalar& tau, RealScalar& beta) const;
+  template <typename EssentialPart>
+  EIGEN_DEVICE_FUNC void applyHouseholderOnTheLeft(const EssentialPart& essential, const Scalar& tau,
+                                                   Scalar* workspace);
+  template <typename EssentialPart>
+  EIGEN_DEVICE_FUNC void applyHouseholderOnTheRight(const EssentialPart& essential, const Scalar& tau,
+                                                    Scalar* workspace);
+
+  ///////// Jacobi module /////////
+
+  template <typename OtherScalar>
+  EIGEN_DEVICE_FUNC void applyOnTheLeft(Index p, Index q, const JacobiRotation<OtherScalar>& j);
+  template <typename OtherScalar>
+  EIGEN_DEVICE_FUNC void applyOnTheRight(Index p, Index q, const JacobiRotation<OtherScalar>& j);
+
+  ///////// SparseCore module /////////
+
+  template <typename OtherDerived>
+  EIGEN_STRONG_INLINE const typename SparseMatrixBase<OtherDerived>::template CwiseProductDenseReturnType<Derived>::Type
+  cwiseProduct(const SparseMatrixBase<OtherDerived>& other) const {
+    return other.cwiseProduct(derived());
+  }
+
+  ///////// MatrixFunctions module /////////
+
+  typedef typename internal::stem_function<Scalar>::type StemFunction;
+#define EIGEN_MATRIX_FUNCTION(ReturnType, Name, Description)                                                        \
+  /** \returns an expression of the matrix Description of \c *this. \brief This function requires the <a            \
+   * href="unsupported/group__MatrixFunctions__Module.html"> unsupported MatrixFunctions module</a>. To compute the \
+   * coefficient-wise Description use ArrayBase::##Name . */                                                        \
+  const ReturnType<Derived> Name() const;
+#define EIGEN_MATRIX_FUNCTION_1(ReturnType, Name, Description, Argument)                                            \
+  /** \returns an expression of the matrix Description of \c *this. \brief This function requires the <a            \
+   * href="unsupported/group__MatrixFunctions__Module.html"> unsupported MatrixFunctions module</a>. To compute the \
+   * coefficient-wise Description use ArrayBase::##Name . */                                                        \
+  const ReturnType<Derived> Name(Argument) const;
+
+  EIGEN_MATRIX_FUNCTION(MatrixExponentialReturnValue, exp, exponential)
+  /** \brief Helper function for the <a href="unsupported/group__MatrixFunctions__Module.html"> unsupported
+   * MatrixFunctions module</a>.*/
+  const MatrixFunctionReturnValue<Derived> matrixFunction(StemFunction f) const;
+  EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, cosh, hyperbolic cosine)
+  EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, sinh, hyperbolic sine)
+  EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, atanh, inverse hyperbolic cosine)
+  EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, acosh, inverse hyperbolic cosine)
+  EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, asinh, inverse hyperbolic sine)
+  EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, cos, cosine)
+  EIGEN_MATRIX_FUNCTION(MatrixFunctionReturnValue, sin, sine)
+  EIGEN_MATRIX_FUNCTION(MatrixSquareRootReturnValue, sqrt, square root)
+  EIGEN_MATRIX_FUNCTION(MatrixLogarithmReturnValue, log, logarithm)
+  EIGEN_MATRIX_FUNCTION_1(MatrixPowerReturnValue, pow, power to \c p, const RealScalar& p)
+  EIGEN_MATRIX_FUNCTION_1(MatrixComplexPowerReturnValue, pow, power to \c p, const std::complex<RealScalar>& p)
+
+ protected:
+  EIGEN_DEFAULT_COPY_CONSTRUCTOR(MatrixBase)
+  EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(MatrixBase)
+
+ private:
+  EIGEN_DEVICE_FUNC explicit MatrixBase(int);
+  EIGEN_DEVICE_FUNC MatrixBase(int, int);
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC explicit MatrixBase(const MatrixBase<OtherDerived>&);
+
+ protected:
+  // mixing arrays and matrices is not legal
+  template <typename OtherDerived>
+  Derived& operator+=(const ArrayBase<OtherDerived>&) {
+    EIGEN_STATIC_ASSERT(std::ptrdiff_t(sizeof(typename OtherDerived::Scalar)) == -1,
+                        YOU_CANNOT_MIX_ARRAYS_AND_MATRICES);
+    return *this;
+  }
+  // mixing arrays and matrices is not legal
+  template <typename OtherDerived>
+  Derived& operator-=(const ArrayBase<OtherDerived>&) {
+    EIGEN_STATIC_ASSERT(std::ptrdiff_t(sizeof(typename OtherDerived::Scalar)) == -1,
+                        YOU_CANNOT_MIX_ARRAYS_AND_MATRICES);
+    return *this;
+  }
+};
+
+/***************************************************************************
+ * Implementation of matrix base methods
+ ***************************************************************************/
+
+/** replaces \c *this by \c *this * \a other.
+ *
+ * \returns a reference to \c *this
+ *
+ * Example: \include MatrixBase_applyOnTheRight.cpp
+ * Output: \verbinclude MatrixBase_applyOnTheRight.out
+ */
+template <typename Derived>
+template <typename OtherDerived>
+inline Derived& MatrixBase<Derived>::operator*=(const EigenBase<OtherDerived>& other) {
+  other.derived().applyThisOnTheRight(derived());
+  return derived();
+}
+
+/** replaces \c *this by \c *this * \a other. It is equivalent to MatrixBase::operator*=().
+ *
+ * Example: \include MatrixBase_applyOnTheRight.cpp
+ * Output: \verbinclude MatrixBase_applyOnTheRight.out
+ */
+template <typename Derived>
+template <typename OtherDerived>
+inline void MatrixBase<Derived>::applyOnTheRight(const EigenBase<OtherDerived>& other) {
+  other.derived().applyThisOnTheRight(derived());
+}
+
+/** replaces \c *this by \a other * \c *this.
+ *
+ * Example: \include MatrixBase_applyOnTheLeft.cpp
+ * Output: \verbinclude MatrixBase_applyOnTheLeft.out
+ */
+template <typename Derived>
+template <typename OtherDerived>
+inline void MatrixBase<Derived>::applyOnTheLeft(const EigenBase<OtherDerived>& other) {
+  other.derived().applyThisOnTheLeft(derived());
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_MATRIXBASE_H

include/Eigen/src/Core/NestByValue.h

@@ -0,0 +1,91 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_NESTBYVALUE_H
+#define EIGEN_NESTBYVALUE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename ExpressionType>
+struct traits<NestByValue<ExpressionType> > : public traits<ExpressionType> {
+  enum { Flags = traits<ExpressionType>::Flags & ~NestByRefBit };
+};
+}  // namespace internal
+
+/** \class NestByValue
+ * \ingroup Core_Module
+ *
+ * \brief Expression which must be nested by value
+ *
+ * \tparam ExpressionType the type of the object of which we are requiring nesting-by-value
+ *
+ * This class is the return type of MatrixBase::nestByValue()
+ * and most of the time this is the only way it is used.
+ *
+ * \sa MatrixBase::nestByValue()
+ */
+template <typename ExpressionType>
+class NestByValue : public internal::dense_xpr_base<NestByValue<ExpressionType> >::type {
+ public:
+  typedef typename internal::dense_xpr_base<NestByValue>::type Base;
+  static constexpr bool HasDirectAccess = internal::has_direct_access<ExpressionType>::ret;
+
+  EIGEN_DENSE_PUBLIC_INTERFACE(NestByValue)
+
+  EIGEN_DEVICE_FUNC explicit inline NestByValue(const ExpressionType& matrix) : m_expression(matrix) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_expression.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_expression.cols(); }
+
+  EIGEN_DEVICE_FUNC operator const ExpressionType&() const { return m_expression; }
+
+  EIGEN_DEVICE_FUNC const ExpressionType& nestedExpression() const { return m_expression; }
+
+  EIGEN_DEVICE_FUNC typename std::enable_if<HasDirectAccess, const Scalar*>::type data() const {
+    return m_expression.data();
+  }
+
+  EIGEN_DEVICE_FUNC typename std::enable_if<HasDirectAccess, Index>::type innerStride() const {
+    return m_expression.innerStride();
+  }
+
+  EIGEN_DEVICE_FUNC typename std::enable_if<HasDirectAccess, Index>::type outerStride() const {
+    return m_expression.outerStride();
+  }
+
+ protected:
+  const ExpressionType m_expression;
+};
+
+/** \returns an expression of the temporary version of *this.
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline const NestByValue<Derived> DenseBase<Derived>::nestByValue() const {
+  return NestByValue<Derived>(derived());
+}
+
+namespace internal {
+
+// Evaluator of Solve -> eval into a temporary
+template <typename ArgType>
+struct evaluator<NestByValue<ArgType> > : public evaluator<ArgType> {
+  typedef evaluator<ArgType> Base;
+
+  EIGEN_DEVICE_FUNC explicit evaluator(const NestByValue<ArgType>& xpr) : Base(xpr.nestedExpression()) {}
+};
+}  // namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_NESTBYVALUE_H

include/Eigen/src/Core/NoAlias.h

@@ -0,0 +1,102 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_NOALIAS_H
+#define EIGEN_NOALIAS_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class NoAlias
+ * \ingroup Core_Module
+ *
+ * \brief Pseudo expression providing an operator = assuming no aliasing
+ *
+ * \tparam ExpressionType the type of the object on which to do the lazy assignment
+ *
+ * This class represents an expression with special assignment operators
+ * assuming no aliasing between the target expression and the source expression.
+ * More precisely it alloas to bypass the EvalBeforeAssignBit flag of the source expression.
+ * It is the return type of MatrixBase::noalias()
+ * and most of the time this is the only way it is used.
+ *
+ * \sa MatrixBase::noalias()
+ */
+template <typename ExpressionType, template <typename> class StorageBase>
+class NoAlias {
+ public:
+  typedef typename ExpressionType::Scalar Scalar;
+
+  EIGEN_DEVICE_FUNC explicit NoAlias(ExpressionType& expression) : m_expression(expression) {}
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ExpressionType& operator=(const StorageBase<OtherDerived>& other) {
+    call_assignment_no_alias(m_expression, other.derived(),
+                             internal::assign_op<Scalar, typename OtherDerived::Scalar>());
+    return m_expression;
+  }
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ExpressionType& operator+=(const StorageBase<OtherDerived>& other) {
+    call_assignment_no_alias(m_expression, other.derived(),
+                             internal::add_assign_op<Scalar, typename OtherDerived::Scalar>());
+    return m_expression;
+  }
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ExpressionType& operator-=(const StorageBase<OtherDerived>& other) {
+    call_assignment_no_alias(m_expression, other.derived(),
+                             internal::sub_assign_op<Scalar, typename OtherDerived::Scalar>());
+    return m_expression;
+  }
+
+  EIGEN_DEVICE_FUNC ExpressionType& expression() const { return m_expression; }
+
+ protected:
+  ExpressionType& m_expression;
+};
+
+/** \returns a pseudo expression of \c *this with an operator= assuming
+ * no aliasing between \c *this and the source expression.
+ *
+ * More precisely, noalias() allows to bypass the EvalBeforeAssignBit flag.
+ * Currently, even though several expressions may alias, only product
+ * expressions have this flag. Therefore, noalias() is only useful when
+ * the source expression contains a matrix product.
+ *
+ * Here are some examples where noalias is useful:
+ * \code
+ * D.noalias()  = A * B;
+ * D.noalias() += A.transpose() * B;
+ * D.noalias() -= 2 * A * B.adjoint();
+ * \endcode
+ *
+ * On the other hand the following example will lead to a \b wrong result:
+ * \code
+ * A.noalias() = A * B;
+ * \endcode
+ * because the result matrix A is also an operand of the matrix product. Therefore,
+ * there is no alternative than evaluating A * B in a temporary, that is the default
+ * behavior when you write:
+ * \code
+ * A = A * B;
+ * \endcode
+ *
+ * \sa class NoAlias
+ */
+template <typename Derived>
+NoAlias<Derived, MatrixBase> EIGEN_DEVICE_FUNC MatrixBase<Derived>::noalias() {
+  return NoAlias<Derived, Eigen::MatrixBase>(derived());
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_NOALIAS_H

include/Eigen/src/Core/NumTraits.h

@@ -0,0 +1,328 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_NUMTRAITS_H
+#define EIGEN_NUMTRAITS_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+// default implementation of digits(), based on numeric_limits if specialized,
+// 0 for integer types, and log2(epsilon()) otherwise.
+template <typename T, bool use_numeric_limits = std::numeric_limits<T>::is_specialized,
+          bool is_integer = NumTraits<T>::IsInteger>
+struct default_digits_impl {
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return std::numeric_limits<T>::digits; }
+};
+
+template <typename T>
+struct default_digits_impl<T, false, false>  // Floating point
+{
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() {
+    using std::ceil;
+    using std::log2;
+    typedef typename NumTraits<T>::Real Real;
+    return int(ceil(-log2(NumTraits<Real>::epsilon())));
+  }
+};
+
+template <typename T>
+struct default_digits_impl<T, false, true>  // Integer
+{
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return 0; }
+};
+
+// default implementation of digits10(), based on numeric_limits if specialized,
+// 0 for integer types, and floor((digits()-1)*log10(2)) otherwise.
+template <typename T, bool use_numeric_limits = std::numeric_limits<T>::is_specialized,
+          bool is_integer = NumTraits<T>::IsInteger>
+struct default_digits10_impl {
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return std::numeric_limits<T>::digits10; }
+};
+
+template <typename T>
+struct default_digits10_impl<T, false, false>  // Floating point
+{
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() {
+    using std::floor;
+    using std::log10;
+    typedef typename NumTraits<T>::Real Real;
+    return int(floor((internal::default_digits_impl<Real>::run() - 1) * log10(2)));
+  }
+};
+
+template <typename T>
+struct default_digits10_impl<T, false, true>  // Integer
+{
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return 0; }
+};
+
+// default implementation of max_digits10(), based on numeric_limits if specialized,
+// 0 for integer types, and log10(2) * digits() + 1 otherwise.
+template <typename T, bool use_numeric_limits = std::numeric_limits<T>::is_specialized,
+          bool is_integer = NumTraits<T>::IsInteger>
+struct default_max_digits10_impl {
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return std::numeric_limits<T>::max_digits10; }
+};
+
+template <typename T>
+struct default_max_digits10_impl<T, false, false>  // Floating point
+{
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() {
+    using std::ceil;
+    using std::log10;
+    typedef typename NumTraits<T>::Real Real;
+    return int(ceil(internal::default_digits_impl<Real>::run() * log10(2) + 1));
+  }
+};
+
+template <typename T>
+struct default_max_digits10_impl<T, false, true>  // Integer
+{
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static int run() { return 0; }
+};
+
+}  // end namespace internal
+
+namespace numext {
+/** \internal bit-wise cast without changing the underlying bit representation. */
+
+// TODO: Replace by std::bit_cast (available in C++20)
+template <typename Tgt, typename Src>
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Tgt bit_cast(const Src& src) {
+  // The behaviour of memcpy is not specified for non-trivially copyable types
+  EIGEN_STATIC_ASSERT(std::is_trivially_copyable<Src>::value, THIS_TYPE_IS_NOT_SUPPORTED)
+  EIGEN_STATIC_ASSERT(std::is_trivially_copyable<Tgt>::value && std::is_default_constructible<Tgt>::value,
+                      THIS_TYPE_IS_NOT_SUPPORTED)
+  EIGEN_STATIC_ASSERT(sizeof(Src) == sizeof(Tgt), THIS_TYPE_IS_NOT_SUPPORTED)
+
+  Tgt tgt;
+  // Load src into registers first. This allows the memcpy to be elided by CUDA.
+  const Src staged = src;
+  EIGEN_USING_STD(memcpy)
+  memcpy(static_cast<void*>(&tgt), static_cast<const void*>(&staged), sizeof(Tgt));
+  return tgt;
+}
+}  // namespace numext
+
+/** \class NumTraits
+ * \ingroup Core_Module
+ *
+ * \brief Holds information about the various numeric (i.e. scalar) types allowed by Eigen.
+ *
+ * \tparam T the numeric type at hand
+ *
+ * This class stores enums, typedefs and static methods giving information about a numeric type.
+ *
+ * The provided data consists of:
+ * \li A typedef \c Real, giving the "real part" type of \a T. If \a T is already real,
+ *     then \c Real is just a typedef to \a T. If \a T is \c std::complex<U> then \c Real
+ *     is a typedef to \a U.
+ * \li A typedef \c NonInteger, giving the type that should be used for operations producing non-integral values,
+ *     such as quotients, square roots, etc. If \a T is a floating-point type, then this typedef just gives
+ *     \a T again. Note however that many Eigen functions such as internal::sqrt simply refuse to
+ *     take integers. Outside of a few cases, Eigen doesn't do automatic type promotion. Thus, this typedef is
+ *     only intended as a helper for code that needs to explicitly promote types.
+ * \li A typedef \c Literal giving the type to use for numeric literals such as "2" or "0.5". For instance, for \c
+ * std::complex<U>, Literal is defined as \c U. Of course, this type must be fully compatible with \a T. In doubt, just
+ * use \a T here. \li A typedef \a Nested giving the type to use to nest a value inside of the expression tree. If you
+ * don't know what this means, just use \a T here. \li An enum value \a IsComplex. It is equal to 1 if \a T is a \c
+ * std::complex type, and to 0 otherwise. \li An enum value \a IsInteger. It is equal to \c 1 if \a T is an integer type
+ * such as \c int, and to \c 0 otherwise. \li Enum values ReadCost, AddCost and MulCost representing a rough estimate of
+ * the number of CPU cycles needed to by move / add / mul instructions respectively, assuming the data is already stored
+ * in CPU registers. Stay vague here. No need to do architecture-specific stuff. If you don't know what this means, just
+ * use \c Eigen::HugeCost. \li An enum value \a IsSigned. It is equal to \c 1 if \a T is a signed type and to 0 if \a T
+ * is unsigned. \li An enum value \a RequireInitialization. It is equal to \c 1 if the constructor of the numeric type
+ * \a T must be called, and to 0 if it is safe not to call it. Default is 0 if \a T is an arithmetic type, and 1
+ * otherwise. \li An epsilon() function which, unlike <a
+ * href="http://en.cppreference.com/w/cpp/types/numeric_limits/epsilon">std::numeric_limits::epsilon()</a>, it returns a
+ * \a Real instead of a \a T. \li A dummy_precision() function returning a weak epsilon value. It is mainly used as a
+ * default value by the fuzzy comparison operators. \li highest() and lowest() functions returning the highest and
+ * lowest possible values respectively. \li digits() function returning the number of radix digits (non-sign digits for
+ * integers, mantissa for floating-point). This is the analogue of <a
+ * href="http://en.cppreference.com/w/cpp/types/numeric_limits/digits">std::numeric_limits<T>::digits</a> which is used
+ * as the default implementation if specialized. \li digits10() function returning the number of decimal digits that can
+ * be represented without change. This is the analogue of <a
+ * href="http://en.cppreference.com/w/cpp/types/numeric_limits/digits10">std::numeric_limits<T>::digits10</a> which is
+ * used as the default implementation if specialized. \li max_digits10() function returning the number of decimal digits
+ * required to uniquely represent all distinct values of the type. This is the analogue of <a
+ * href="http://en.cppreference.com/w/cpp/types/numeric_limits/max_digits10">std::numeric_limits<T>::max_digits10</a>
+ *     which is used as the default implementation if specialized.
+ * \li min_exponent() and max_exponent() functions returning the highest and lowest possible values, respectively,
+ *     such that the radix raised to the power exponent-1 is a normalized floating-point number.  These are equivalent
+ * to <a
+ * href="http://en.cppreference.com/w/cpp/types/numeric_limits/min_exponent">std::numeric_limits<T>::min_exponent</a>/
+ *     <a
+ * href="http://en.cppreference.com/w/cpp/types/numeric_limits/max_exponent">std::numeric_limits<T>::max_exponent</a>.
+ * \li infinity() function returning a representation of positive infinity, if available.
+ * \li quiet_NaN function returning a non-signaling "not-a-number", if available.
+ */
+
+template <typename T>
+struct GenericNumTraits {
+  enum {
+    IsInteger = std::numeric_limits<T>::is_integer,
+    IsSigned = std::numeric_limits<T>::is_signed,
+    IsComplex = 0,
+    RequireInitialization = internal::is_arithmetic<T>::value ? 0 : 1,
+    ReadCost = 1,
+    AddCost = 1,
+    MulCost = 1
+  };
+
+  typedef T Real;
+  typedef std::conditional_t<IsInteger, std::conditional_t<sizeof(T) <= 2, float, double>, T> NonInteger;
+  typedef T Nested;
+  typedef T Literal;
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline Real epsilon() { return numext::numeric_limits<T>::epsilon(); }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int digits10() { return internal::default_digits10_impl<T>::run(); }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int max_digits10() {
+    return internal::default_max_digits10_impl<T>::run();
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int digits() { return internal::default_digits_impl<T>::run(); }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int min_exponent() { return numext::numeric_limits<T>::min_exponent; }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int max_exponent() { return numext::numeric_limits<T>::max_exponent; }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline Real dummy_precision() {
+    // make sure to override this for floating-point types
+    return Real(0);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline T highest() { return (numext::numeric_limits<T>::max)(); }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline T lowest() { return (numext::numeric_limits<T>::lowest)(); }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline T infinity() { return numext::numeric_limits<T>::infinity(); }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline T quiet_NaN() { return numext::numeric_limits<T>::quiet_NaN(); }
+};
+
+template <typename T>
+struct NumTraits : GenericNumTraits<T> {};
+
+template <>
+struct NumTraits<float> : GenericNumTraits<float> {
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline float dummy_precision() { return 1e-5f; }
+};
+
+template <>
+struct NumTraits<double> : GenericNumTraits<double> {
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline double dummy_precision() { return 1e-12; }
+};
+
+// GPU devices treat `long double` as `double`.
+#ifndef EIGEN_GPU_COMPILE_PHASE
+template <>
+struct NumTraits<long double> : GenericNumTraits<long double> {
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline long double dummy_precision() {
+    return static_cast<long double>(1e-15l);
+  }
+
+#if defined(EIGEN_ARCH_PPC) && (__LDBL_MANT_DIG__ == 106)
+  // PowerPC double double causes issues with some values
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline long double epsilon() {
+    // 2^(-(__LDBL_MANT_DIG__)+1)
+    return static_cast<long double>(2.4651903288156618919116517665087e-32l);
+  }
+#endif
+};
+#endif
+
+template <typename Real_>
+struct NumTraits<std::complex<Real_> > : GenericNumTraits<std::complex<Real_> > {
+  typedef Real_ Real;
+  typedef typename NumTraits<Real_>::Literal Literal;
+  enum {
+    IsComplex = 1,
+    IsSigned = NumTraits<Real_>::IsSigned,
+    RequireInitialization = NumTraits<Real_>::RequireInitialization,
+    ReadCost = 2 * NumTraits<Real_>::ReadCost,
+    AddCost = 2 * NumTraits<Real>::AddCost,
+    MulCost = 4 * NumTraits<Real>::MulCost + 2 * NumTraits<Real>::AddCost
+  };
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline Real epsilon() { return NumTraits<Real>::epsilon(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline Real dummy_precision() { return NumTraits<Real>::dummy_precision(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int digits10() { return NumTraits<Real>::digits10(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline int max_digits10() { return NumTraits<Real>::max_digits10(); }
+};
+
+template <typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols>
+struct NumTraits<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> > {
+  typedef Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> ArrayType;
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+  typedef Array<RealScalar, Rows, Cols, Options, MaxRows, MaxCols> Real;
+  typedef typename NumTraits<Scalar>::NonInteger NonIntegerScalar;
+  typedef Array<NonIntegerScalar, Rows, Cols, Options, MaxRows, MaxCols> NonInteger;
+  typedef ArrayType& Nested;
+  typedef typename NumTraits<Scalar>::Literal Literal;
+
+  enum {
+    IsComplex = NumTraits<Scalar>::IsComplex,
+    IsInteger = NumTraits<Scalar>::IsInteger,
+    IsSigned = NumTraits<Scalar>::IsSigned,
+    RequireInitialization = 1,
+    ReadCost = ArrayType::SizeAtCompileTime == Dynamic
+                   ? HugeCost
+                   : ArrayType::SizeAtCompileTime * int(NumTraits<Scalar>::ReadCost),
+    AddCost = ArrayType::SizeAtCompileTime == Dynamic ? HugeCost
+                                                      : ArrayType::SizeAtCompileTime * int(NumTraits<Scalar>::AddCost),
+    MulCost = ArrayType::SizeAtCompileTime == Dynamic ? HugeCost
+                                                      : ArrayType::SizeAtCompileTime * int(NumTraits<Scalar>::MulCost)
+  };
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline RealScalar epsilon() { return NumTraits<RealScalar>::epsilon(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static inline RealScalar dummy_precision() {
+    return NumTraits<RealScalar>::dummy_precision();
+  }
+
+  EIGEN_CONSTEXPR
+  static inline int digits10() { return NumTraits<Scalar>::digits10(); }
+  EIGEN_CONSTEXPR
+  static inline int max_digits10() { return NumTraits<Scalar>::max_digits10(); }
+};
+
+template <>
+struct NumTraits<std::string> : GenericNumTraits<std::string> {
+  enum { RequireInitialization = 1, ReadCost = HugeCost, AddCost = HugeCost, MulCost = HugeCost };
+
+  EIGEN_CONSTEXPR
+  static inline int digits10() { return 0; }
+  EIGEN_CONSTEXPR
+  static inline int max_digits10() { return 0; }
+
+ private:
+  static inline std::string epsilon();
+  static inline std::string dummy_precision();
+  static inline std::string lowest();
+  static inline std::string highest();
+  static inline std::string infinity();
+  static inline std::string quiet_NaN();
+};
+
+// Empty specialization for void to allow template specialization based on NumTraits<T>::Real with T==void and SFINAE.
+template <>
+struct NumTraits<void> {};
+
+template <>
+struct NumTraits<bool> : GenericNumTraits<bool> {};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_NUMTRAITS_H

include/Eigen/src/Core/PartialReduxEvaluator.h

@@ -0,0 +1,209 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2011-2018 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_PARTIALREDUX_H
+#define EIGEN_PARTIALREDUX_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+/***************************************************************************
+ *
+ * This file provides evaluators for partial reductions.
+ * There are two modes:
+ *
+ *  - scalar path: simply calls the respective function on the column or row.
+ *    -> nothing special here, all the tricky part is handled by the return
+ *       types of VectorwiseOp's members. They embed the functor calling the
+ *       respective DenseBase's member function.
+ *
+ *  - vectorized path: implements a packet-wise reductions followed by
+ *    some (optional) processing of the outcome, e.g., division by n for mean.
+ *
+ * For the vectorized path let's observe that the packet-size and outer-unrolling
+ * are both decided by the assignment logic. So all we have to do is to decide
+ * on the inner unrolling.
+ *
+ * For the unrolling, we can reuse "internal::redux_vec_unroller" from Redux.h,
+ * but be need to be careful to specify correct increment.
+ *
+ ***************************************************************************/
+
+/* logic deciding a strategy for unrolling of vectorized paths */
+template <typename Func, typename Evaluator>
+struct packetwise_redux_traits {
+  enum {
+    OuterSize = int(Evaluator::IsRowMajor) ? Evaluator::RowsAtCompileTime : Evaluator::ColsAtCompileTime,
+    Cost = OuterSize == Dynamic ? HugeCost
+                                : OuterSize * Evaluator::CoeffReadCost + (OuterSize - 1) * functor_traits<Func>::Cost,
+    Unrolling = Cost <= EIGEN_UNROLLING_LIMIT ? CompleteUnrolling : NoUnrolling
+  };
+};
+
+/* Value to be returned when size==0 , by default let's return 0 */
+template <typename PacketType, typename Func>
+EIGEN_DEVICE_FUNC PacketType packetwise_redux_empty_value(const Func&) {
+  const typename unpacket_traits<PacketType>::type zero(0);
+  return pset1<PacketType>(zero);
+}
+
+/* For products the default is 1 */
+template <typename PacketType, typename Scalar>
+EIGEN_DEVICE_FUNC PacketType packetwise_redux_empty_value(const scalar_product_op<Scalar, Scalar>&) {
+  return pset1<PacketType>(Scalar(1));
+}
+
+/* Perform the actual reduction */
+template <typename Func, typename Evaluator, int Unrolling = packetwise_redux_traits<Func, Evaluator>::Unrolling>
+struct packetwise_redux_impl;
+
+/* Perform the actual reduction with unrolling */
+template <typename Func, typename Evaluator>
+struct packetwise_redux_impl<Func, Evaluator, CompleteUnrolling> {
+  typedef redux_novec_unroller<Func, Evaluator, 0, Evaluator::SizeAtCompileTime> Base;
+  typedef typename Evaluator::Scalar Scalar;
+
+  template <typename PacketType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator& eval, const Func& func, Index /*size*/) {
+    return redux_vec_unroller<Func, Evaluator, 0,
+                              packetwise_redux_traits<Func, Evaluator>::OuterSize>::template run<PacketType>(eval,
+                                                                                                             func);
+  }
+};
+
+/* Add a specialization of redux_vec_unroller for size==0 at compiletime.
+ * This specialization is not required for general reductions, which is
+ * why it is defined here.
+ */
+template <typename Func, typename Evaluator, Index Start>
+struct redux_vec_unroller<Func, Evaluator, Start, 0> {
+  template <typename PacketType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator&, const Func& f) {
+    return packetwise_redux_empty_value<PacketType>(f);
+  }
+};
+
+/* Perform the actual reduction for dynamic sizes */
+template <typename Func, typename Evaluator>
+struct packetwise_redux_impl<Func, Evaluator, NoUnrolling> {
+  typedef typename Evaluator::Scalar Scalar;
+  typedef typename redux_traits<Func, Evaluator>::PacketType PacketScalar;
+
+  template <typename PacketType>
+  EIGEN_DEVICE_FUNC static PacketType run(const Evaluator& eval, const Func& func, Index size) {
+    if (size == 0) return packetwise_redux_empty_value<PacketType>(func);
+
+    const Index size4 = (size - 1) & (~3);
+    PacketType p = eval.template packetByOuterInner<Unaligned, PacketType>(0, 0);
+    Index i = 1;
+    // This loop is optimized for instruction pipelining:
+    // - each iteration generates two independent instructions
+    // - thanks to branch prediction and out-of-order execution we have independent instructions across loops
+    for (; i < size4; i += 4)
+      p = func.packetOp(
+          p, func.packetOp(func.packetOp(eval.template packetByOuterInner<Unaligned, PacketType>(i + 0, 0),
+                                         eval.template packetByOuterInner<Unaligned, PacketType>(i + 1, 0)),
+                           func.packetOp(eval.template packetByOuterInner<Unaligned, PacketType>(i + 2, 0),
+                                         eval.template packetByOuterInner<Unaligned, PacketType>(i + 3, 0))));
+    for (; i < size; ++i) p = func.packetOp(p, eval.template packetByOuterInner<Unaligned, PacketType>(i, 0));
+    return p;
+  }
+};
+
+template <typename ArgType, typename MemberOp, int Direction>
+struct evaluator<PartialReduxExpr<ArgType, MemberOp, Direction> >
+    : evaluator_base<PartialReduxExpr<ArgType, MemberOp, Direction> > {
+  typedef PartialReduxExpr<ArgType, MemberOp, Direction> XprType;
+  typedef typename internal::nested_eval<ArgType, 1>::type ArgTypeNested;
+  typedef add_const_on_value_type_t<ArgTypeNested> ConstArgTypeNested;
+  typedef internal::remove_all_t<ArgTypeNested> ArgTypeNestedCleaned;
+  typedef typename ArgType::Scalar InputScalar;
+  typedef typename XprType::Scalar Scalar;
+  enum {
+    TraversalSize = Direction == int(Vertical) ? int(ArgType::RowsAtCompileTime) : int(ArgType::ColsAtCompileTime)
+  };
+  typedef typename MemberOp::template Cost<int(TraversalSize)> CostOpType;
+  enum {
+    CoeffReadCost = TraversalSize == Dynamic ? HugeCost
+                    : TraversalSize == 0
+                        ? 1
+                        : int(TraversalSize) * int(evaluator<ArgType>::CoeffReadCost) + int(CostOpType::value),
+
+    ArgFlags_ = evaluator<ArgType>::Flags,
+
+    Vectorizable_ = bool(int(ArgFlags_) & PacketAccessBit) && bool(MemberOp::Vectorizable) &&
+                    (Direction == int(Vertical) ? bool(ArgFlags_ & RowMajorBit) : (ArgFlags_ & RowMajorBit) == 0) &&
+                    (TraversalSize != 0),
+
+    Flags = (traits<XprType>::Flags & RowMajorBit) | (evaluator<ArgType>::Flags & (HereditaryBits & (~RowMajorBit))) |
+            (Vectorizable_ ? PacketAccessBit : 0) | LinearAccessBit,
+
+    Alignment = 0  // FIXME this will need to be improved once PartialReduxExpr is vectorized
+  };
+
+  EIGEN_DEVICE_FUNC explicit evaluator(const XprType xpr) : m_arg(xpr.nestedExpression()), m_functor(xpr.functor()) {
+    EIGEN_INTERNAL_CHECK_COST_VALUE(TraversalSize == Dynamic ? HugeCost
+                                                             : (TraversalSize == 0 ? 1 : int(CostOpType::value)));
+    EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
+  }
+
+  typedef typename XprType::CoeffReturnType CoeffReturnType;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(Index i, Index j) const {
+    return coeff(Direction == Vertical ? j : i);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(Index index) const {
+    return m_functor(m_arg.template subVector<DirectionType(Direction)>(index));
+  }
+
+  template <int LoadMode, typename PacketType>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index i, Index j) const {
+    return packet<LoadMode, PacketType>(Direction == Vertical ? j : i);
+  }
+
+  template <int LoadMode, typename PacketType>
+  EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC PacketType packet(Index idx) const {
+    enum { PacketSize = internal::unpacket_traits<PacketType>::size };
+    typedef Block<const ArgTypeNestedCleaned, Direction == Vertical ? int(ArgType::RowsAtCompileTime) : int(PacketSize),
+                  Direction == Vertical ? int(PacketSize) : int(ArgType::ColsAtCompileTime), true /* InnerPanel */>
+        PanelType;
+
+    PanelType panel(m_arg, Direction == Vertical ? 0 : idx, Direction == Vertical ? idx : 0,
+                    Direction == Vertical ? m_arg.rows() : Index(PacketSize),
+                    Direction == Vertical ? Index(PacketSize) : m_arg.cols());
+
+    // FIXME
+    // See bug 1612, currently if PacketSize==1 (i.e. complex<double> with 128bits registers) then the storage-order of
+    // panel get reversed and methods like packetByOuterInner do not make sense anymore in this context. So let's just
+    // by pass "vectorization" in this case:
+    if (PacketSize == 1) return internal::pset1<PacketType>(coeff(idx));
+
+    typedef typename internal::redux_evaluator<PanelType> PanelEvaluator;
+    PanelEvaluator panel_eval(panel);
+    typedef typename MemberOp::BinaryOp BinaryOp;
+    PacketType p = internal::packetwise_redux_impl<BinaryOp, PanelEvaluator>::template run<PacketType>(
+        panel_eval, m_functor.binaryFunc(), m_arg.outerSize());
+    return p;
+  }
+
+ protected:
+  ConstArgTypeNested m_arg;
+  const MemberOp m_functor;
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_PARTIALREDUX_H

include/Eigen/src/Core/PermutationMatrix.h

@@ -0,0 +1,552 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2009-2015 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_PERMUTATIONMATRIX_H
+#define EIGEN_PERMUTATIONMATRIX_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+enum PermPermProduct_t { PermPermProduct };
+
+}  // end namespace internal
+
+/** \class PermutationBase
+ * \ingroup Core_Module
+ *
+ * \brief Base class for permutations
+ *
+ * \tparam Derived the derived class
+ *
+ * This class is the base class for all expressions representing a permutation matrix,
+ * internally stored as a vector of integers.
+ * The convention followed here is that if \f$ \sigma \f$ is a permutation, the corresponding permutation matrix
+ * \f$ P_\sigma \f$ is such that if \f$ (e_1,\ldots,e_p) \f$ is the canonical basis, we have:
+ *  \f[ P_\sigma(e_i) = e_{\sigma(i)}. \f]
+ * This convention ensures that for any two permutations \f$ \sigma, \tau \f$, we have:
+ *  \f[ P_{\sigma\circ\tau} = P_\sigma P_\tau. \f]
+ *
+ * Permutation matrices are square and invertible.
+ *
+ * Notice that in addition to the member functions and operators listed here, there also are non-member
+ * operator* to multiply any kind of permutation object with any kind of matrix expression (MatrixBase)
+ * on either side.
+ *
+ * \sa class PermutationMatrix, class PermutationWrapper
+ */
+template <typename Derived>
+class PermutationBase : public EigenBase<Derived> {
+  typedef internal::traits<Derived> Traits;
+  typedef EigenBase<Derived> Base;
+
+ public:
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef typename Traits::IndicesType IndicesType;
+  enum {
+    Flags = Traits::Flags,
+    RowsAtCompileTime = Traits::RowsAtCompileTime,
+    ColsAtCompileTime = Traits::ColsAtCompileTime,
+    MaxRowsAtCompileTime = Traits::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = Traits::MaxColsAtCompileTime
+  };
+  typedef typename Traits::StorageIndex StorageIndex;
+  typedef Matrix<StorageIndex, RowsAtCompileTime, ColsAtCompileTime, 0, MaxRowsAtCompileTime, MaxColsAtCompileTime>
+      DenseMatrixType;
+  typedef PermutationMatrix<IndicesType::SizeAtCompileTime, IndicesType::MaxSizeAtCompileTime, StorageIndex>
+      PlainPermutationType;
+  typedef PlainPermutationType PlainObject;
+  using Base::derived;
+  typedef Inverse<Derived> InverseReturnType;
+  typedef void Scalar;
+#endif
+
+  /** Copies the other permutation into *this */
+  template <typename OtherDerived>
+  Derived& operator=(const PermutationBase<OtherDerived>& other) {
+    indices() = other.indices();
+    return derived();
+  }
+
+  /** Assignment from the Transpositions \a tr */
+  template <typename OtherDerived>
+  Derived& operator=(const TranspositionsBase<OtherDerived>& tr) {
+    setIdentity(tr.size());
+    for (Index k = size() - 1; k >= 0; --k) applyTranspositionOnTheRight(k, tr.coeff(k));
+    return derived();
+  }
+
+  /** \returns the number of rows */
+  inline EIGEN_DEVICE_FUNC Index rows() const { return Index(indices().size()); }
+
+  /** \returns the number of columns */
+  inline EIGEN_DEVICE_FUNC Index cols() const { return Index(indices().size()); }
+
+  /** \returns the size of a side of the respective square matrix, i.e., the number of indices */
+  inline EIGEN_DEVICE_FUNC Index size() const { return Index(indices().size()); }
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  template <typename DenseDerived>
+  void evalTo(MatrixBase<DenseDerived>& other) const {
+    other.setZero();
+    for (Index i = 0; i < rows(); ++i) other.coeffRef(indices().coeff(i), i) = typename DenseDerived::Scalar(1);
+  }
+#endif
+
+  /** \returns a Matrix object initialized from this permutation matrix. Notice that it
+   * is inefficient to return this Matrix object by value. For efficiency, favor using
+   * the Matrix constructor taking EigenBase objects.
+   */
+  DenseMatrixType toDenseMatrix() const { return derived(); }
+
+  /** const version of indices(). */
+  const IndicesType& indices() const { return derived().indices(); }
+  /** \returns a reference to the stored array representing the permutation. */
+  IndicesType& indices() { return derived().indices(); }
+
+  /** Resizes to given size.
+   */
+  inline void resize(Index newSize) { indices().resize(newSize); }
+
+  /** Sets *this to be the identity permutation matrix */
+  void setIdentity() {
+    StorageIndex n = StorageIndex(size());
+    for (StorageIndex i = 0; i < n; ++i) indices().coeffRef(i) = i;
+  }
+
+  /** Sets *this to be the identity permutation matrix of given size.
+   */
+  void setIdentity(Index newSize) {
+    resize(newSize);
+    setIdentity();
+  }
+
+  /** Multiplies *this by the transposition \f$(ij)\f$ on the left.
+   *
+   * \returns a reference to *this.
+   *
+   * \warning This is much slower than applyTranspositionOnTheRight(Index,Index):
+   * this has linear complexity and requires a lot of branching.
+   *
+   * \sa applyTranspositionOnTheRight(Index,Index)
+   */
+  Derived& applyTranspositionOnTheLeft(Index i, Index j) {
+    eigen_assert(i >= 0 && j >= 0 && i < size() && j < size());
+    for (Index k = 0; k < size(); ++k) {
+      if (indices().coeff(k) == i)
+        indices().coeffRef(k) = StorageIndex(j);
+      else if (indices().coeff(k) == j)
+        indices().coeffRef(k) = StorageIndex(i);
+    }
+    return derived();
+  }
+
+  /** Multiplies *this by the transposition \f$(ij)\f$ on the right.
+   *
+   * \returns a reference to *this.
+   *
+   * This is a fast operation, it only consists in swapping two indices.
+   *
+   * \sa applyTranspositionOnTheLeft(Index,Index)
+   */
+  Derived& applyTranspositionOnTheRight(Index i, Index j) {
+    eigen_assert(i >= 0 && j >= 0 && i < size() && j < size());
+    std::swap(indices().coeffRef(i), indices().coeffRef(j));
+    return derived();
+  }
+
+  /** \returns the inverse permutation matrix.
+   *
+   * \note \blank \note_try_to_help_rvo
+   */
+  inline InverseReturnType inverse() const { return InverseReturnType(derived()); }
+  /** \returns the transpose permutation matrix.
+   *
+   * \note \blank \note_try_to_help_rvo
+   */
+  inline InverseReturnType transpose() const { return InverseReturnType(derived()); }
+
+  /**** multiplication helpers to hopefully get RVO ****/
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+ protected:
+  template <typename OtherDerived>
+  void assignTranspose(const PermutationBase<OtherDerived>& other) {
+    for (Index i = 0; i < rows(); ++i) indices().coeffRef(other.indices().coeff(i)) = i;
+  }
+  template <typename Lhs, typename Rhs>
+  void assignProduct(const Lhs& lhs, const Rhs& rhs) {
+    eigen_assert(lhs.cols() == rhs.rows());
+    for (Index i = 0; i < rows(); ++i) indices().coeffRef(i) = lhs.indices().coeff(rhs.indices().coeff(i));
+  }
+#endif
+
+ public:
+  /** \returns the product permutation matrix.
+   *
+   * \note \blank \note_try_to_help_rvo
+   */
+  template <typename Other>
+  inline PlainPermutationType operator*(const PermutationBase<Other>& other) const {
+    return PlainPermutationType(internal::PermPermProduct, derived(), other.derived());
+  }
+
+  /** \returns the product of a permutation with another inverse permutation.
+   *
+   * \note \blank \note_try_to_help_rvo
+   */
+  template <typename Other>
+  inline PlainPermutationType operator*(const InverseImpl<Other, PermutationStorage>& other) const {
+    return PlainPermutationType(internal::PermPermProduct, *this, other.eval());
+  }
+
+  /** \returns the product of an inverse permutation with another permutation.
+   *
+   * \note \blank \note_try_to_help_rvo
+   */
+  template <typename Other>
+  friend inline PlainPermutationType operator*(const InverseImpl<Other, PermutationStorage>& other,
+                                               const PermutationBase& perm) {
+    return PlainPermutationType(internal::PermPermProduct, other.eval(), perm);
+  }
+
+  /** \returns the determinant of the permutation matrix, which is either 1 or -1 depending on the parity of the
+   * permutation.
+   *
+   * This function is O(\c n) procedure allocating a buffer of \c n booleans.
+   */
+  Index determinant() const {
+    Index res = 1;
+    Index n = size();
+    Matrix<bool, RowsAtCompileTime, 1, 0, MaxRowsAtCompileTime> mask(n);
+    mask.fill(false);
+    Index r = 0;
+    while (r < n) {
+      // search for the next seed
+      while (r < n && mask[r]) r++;
+      if (r >= n) break;
+      // we got one, let's follow it until we are back to the seed
+      Index k0 = r++;
+      mask.coeffRef(k0) = true;
+      for (Index k = indices().coeff(k0); k != k0; k = indices().coeff(k)) {
+        mask.coeffRef(k) = true;
+        res = -res;
+      }
+    }
+    return res;
+  }
+
+ protected:
+};
+
+namespace internal {
+template <int SizeAtCompileTime, int MaxSizeAtCompileTime, typename StorageIndex_>
+struct traits<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_> >
+    : traits<
+          Matrix<StorageIndex_, SizeAtCompileTime, SizeAtCompileTime, 0, MaxSizeAtCompileTime, MaxSizeAtCompileTime> > {
+  typedef PermutationStorage StorageKind;
+  typedef Matrix<StorageIndex_, SizeAtCompileTime, 1, 0, MaxSizeAtCompileTime, 1> IndicesType;
+  typedef StorageIndex_ StorageIndex;
+  typedef void Scalar;
+};
+}  // namespace internal
+
+/** \class PermutationMatrix
+ * \ingroup Core_Module
+ *
+ * \brief Permutation matrix
+ *
+ * \tparam SizeAtCompileTime the number of rows/cols, or Dynamic
+ * \tparam MaxSizeAtCompileTime the maximum number of rows/cols, or Dynamic. This optional parameter defaults to
+ * SizeAtCompileTime. Most of the time, you should not have to specify it. \tparam StorageIndex_ the integer type of the
+ * indices
+ *
+ * This class represents a permutation matrix, internally stored as a vector of integers.
+ *
+ * \sa class PermutationBase, class PermutationWrapper, class DiagonalMatrix
+ */
+template <int SizeAtCompileTime, int MaxSizeAtCompileTime, typename StorageIndex_>
+class PermutationMatrix
+    : public PermutationBase<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_> > {
+  typedef PermutationBase<PermutationMatrix> Base;
+  typedef internal::traits<PermutationMatrix> Traits;
+
+ public:
+  typedef const PermutationMatrix& Nested;
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef typename Traits::IndicesType IndicesType;
+  typedef typename Traits::StorageIndex StorageIndex;
+#endif
+
+  inline PermutationMatrix() {}
+
+  /** Constructs an uninitialized permutation matrix of given size.
+   */
+  explicit inline PermutationMatrix(Index size) : m_indices(size) {
+    eigen_internal_assert(size <= NumTraits<StorageIndex>::highest());
+  }
+
+  /** Copy constructor. */
+  template <typename OtherDerived>
+  inline PermutationMatrix(const PermutationBase<OtherDerived>& other) : m_indices(other.indices()) {}
+
+  /** Generic constructor from expression of the indices. The indices
+   * array has the meaning that the permutations sends each integer i to indices[i].
+   *
+   * \warning It is your responsibility to check that the indices array that you passes actually
+   * describes a permutation, i.e., each value between 0 and n-1 occurs exactly once, where n is the
+   * array's size.
+   */
+  template <typename Other>
+  explicit inline PermutationMatrix(const MatrixBase<Other>& indices) : m_indices(indices) {}
+
+  /** Convert the Transpositions \a tr to a permutation matrix */
+  template <typename Other>
+  explicit PermutationMatrix(const TranspositionsBase<Other>& tr) : m_indices(tr.size()) {
+    *this = tr;
+  }
+
+  /** Copies the other permutation into *this */
+  template <typename Other>
+  PermutationMatrix& operator=(const PermutationBase<Other>& other) {
+    m_indices = other.indices();
+    return *this;
+  }
+
+  /** Assignment from the Transpositions \a tr */
+  template <typename Other>
+  PermutationMatrix& operator=(const TranspositionsBase<Other>& tr) {
+    return Base::operator=(tr.derived());
+  }
+
+  /** const version of indices(). */
+  const IndicesType& indices() const { return m_indices; }
+  /** \returns a reference to the stored array representing the permutation. */
+  IndicesType& indices() { return m_indices; }
+
+  /**** multiplication helpers to hopefully get RVO ****/
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  template <typename Other>
+  PermutationMatrix(const InverseImpl<Other, PermutationStorage>& other)
+      : m_indices(other.derived().nestedExpression().size()) {
+    eigen_internal_assert(m_indices.size() <= NumTraits<StorageIndex>::highest());
+    StorageIndex end = StorageIndex(m_indices.size());
+    for (StorageIndex i = 0; i < end; ++i)
+      m_indices.coeffRef(other.derived().nestedExpression().indices().coeff(i)) = i;
+  }
+  template <typename Lhs, typename Rhs>
+  PermutationMatrix(internal::PermPermProduct_t, const Lhs& lhs, const Rhs& rhs) : m_indices(lhs.indices().size()) {
+    Base::assignProduct(lhs, rhs);
+  }
+#endif
+
+ protected:
+  IndicesType m_indices;
+};
+
+namespace internal {
+template <int SizeAtCompileTime, int MaxSizeAtCompileTime, typename StorageIndex_, int PacketAccess_>
+struct traits<Map<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_>, PacketAccess_> >
+    : traits<
+          Matrix<StorageIndex_, SizeAtCompileTime, SizeAtCompileTime, 0, MaxSizeAtCompileTime, MaxSizeAtCompileTime> > {
+  typedef PermutationStorage StorageKind;
+  typedef Map<const Matrix<StorageIndex_, SizeAtCompileTime, 1, 0, MaxSizeAtCompileTime, 1>, PacketAccess_> IndicesType;
+  typedef StorageIndex_ StorageIndex;
+  typedef void Scalar;
+};
+}  // namespace internal
+
+template <int SizeAtCompileTime, int MaxSizeAtCompileTime, typename StorageIndex_, int PacketAccess_>
+class Map<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_>, PacketAccess_>
+    : public PermutationBase<
+          Map<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_>, PacketAccess_> > {
+  typedef PermutationBase<Map> Base;
+  typedef internal::traits<Map> Traits;
+
+ public:
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef typename Traits::IndicesType IndicesType;
+  typedef typename IndicesType::Scalar StorageIndex;
+#endif
+
+  inline Map(const StorageIndex* indicesPtr) : m_indices(indicesPtr) {}
+
+  inline Map(const StorageIndex* indicesPtr, Index size) : m_indices(indicesPtr, size) {}
+
+  /** Copies the other permutation into *this */
+  template <typename Other>
+  Map& operator=(const PermutationBase<Other>& other) {
+    return Base::operator=(other.derived());
+  }
+
+  /** Assignment from the Transpositions \a tr */
+  template <typename Other>
+  Map& operator=(const TranspositionsBase<Other>& tr) {
+    return Base::operator=(tr.derived());
+  }
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** This is a special case of the templated operator=. Its purpose is to
+   * prevent a default operator= from hiding the templated operator=.
+   */
+  Map& operator=(const Map& other) {
+    m_indices = other.m_indices;
+    return *this;
+  }
+#endif
+
+  /** const version of indices(). */
+  const IndicesType& indices() const { return m_indices; }
+  /** \returns a reference to the stored array representing the permutation. */
+  IndicesType& indices() { return m_indices; }
+
+ protected:
+  IndicesType m_indices;
+};
+
+template <typename IndicesType_>
+class TranspositionsWrapper;
+namespace internal {
+template <typename IndicesType_>
+struct traits<PermutationWrapper<IndicesType_> > {
+  typedef PermutationStorage StorageKind;
+  typedef void Scalar;
+  typedef typename IndicesType_::Scalar StorageIndex;
+  typedef IndicesType_ IndicesType;
+  enum {
+    RowsAtCompileTime = IndicesType_::SizeAtCompileTime,
+    ColsAtCompileTime = IndicesType_::SizeAtCompileTime,
+    MaxRowsAtCompileTime = IndicesType::MaxSizeAtCompileTime,
+    MaxColsAtCompileTime = IndicesType::MaxSizeAtCompileTime,
+    Flags = 0
+  };
+};
+}  // namespace internal
+
+/** \class PermutationWrapper
+ * \ingroup Core_Module
+ *
+ * \brief Class to view a vector of integers as a permutation matrix
+ *
+ * \tparam IndicesType_ the type of the vector of integer (can be any compatible expression)
+ *
+ * This class allows to view any vector expression of integers as a permutation matrix.
+ *
+ * \sa class PermutationBase, class PermutationMatrix
+ */
+template <typename IndicesType_>
+class PermutationWrapper : public PermutationBase<PermutationWrapper<IndicesType_> > {
+  typedef PermutationBase<PermutationWrapper> Base;
+  typedef internal::traits<PermutationWrapper> Traits;
+
+ public:
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef typename Traits::IndicesType IndicesType;
+#endif
+
+  inline PermutationWrapper(const IndicesType& indices) : m_indices(indices) {}
+
+  /** const version of indices(). */
+  const internal::remove_all_t<typename IndicesType::Nested>& indices() const { return m_indices; }
+
+ protected:
+  typename IndicesType::Nested m_indices;
+};
+
+/** \returns the matrix with the permutation applied to the columns.
+ */
+template <typename MatrixDerived, typename PermutationDerived>
+EIGEN_DEVICE_FUNC const Product<MatrixDerived, PermutationDerived, AliasFreeProduct> operator*(
+    const MatrixBase<MatrixDerived>& matrix, const PermutationBase<PermutationDerived>& permutation) {
+  return Product<MatrixDerived, PermutationDerived, AliasFreeProduct>(matrix.derived(), permutation.derived());
+}
+
+/** \returns the matrix with the permutation applied to the rows.
+ */
+template <typename PermutationDerived, typename MatrixDerived>
+EIGEN_DEVICE_FUNC const Product<PermutationDerived, MatrixDerived, AliasFreeProduct> operator*(
+    const PermutationBase<PermutationDerived>& permutation, const MatrixBase<MatrixDerived>& matrix) {
+  return Product<PermutationDerived, MatrixDerived, AliasFreeProduct>(permutation.derived(), matrix.derived());
+}
+
+template <typename PermutationType>
+class InverseImpl<PermutationType, PermutationStorage> : public EigenBase<Inverse<PermutationType> > {
+  typedef typename PermutationType::PlainPermutationType PlainPermutationType;
+  typedef internal::traits<PermutationType> PermTraits;
+
+ protected:
+  InverseImpl() {}
+
+ public:
+  typedef Inverse<PermutationType> InverseType;
+  using EigenBase<Inverse<PermutationType> >::derived;
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef typename PermutationType::DenseMatrixType DenseMatrixType;
+  enum {
+    RowsAtCompileTime = PermTraits::RowsAtCompileTime,
+    ColsAtCompileTime = PermTraits::ColsAtCompileTime,
+    MaxRowsAtCompileTime = PermTraits::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = PermTraits::MaxColsAtCompileTime
+  };
+#endif
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  template <typename DenseDerived>
+  void evalTo(MatrixBase<DenseDerived>& other) const {
+    other.setZero();
+    for (Index i = 0; i < derived().rows(); ++i)
+      other.coeffRef(i, derived().nestedExpression().indices().coeff(i)) = typename DenseDerived::Scalar(1);
+  }
+#endif
+
+  /** \return the equivalent permutation matrix */
+  PlainPermutationType eval() const { return derived(); }
+
+  DenseMatrixType toDenseMatrix() const { return derived(); }
+
+  /** \returns the matrix with the inverse permutation applied to the columns.
+   */
+  template <typename OtherDerived>
+  friend const Product<OtherDerived, InverseType, AliasFreeProduct> operator*(const MatrixBase<OtherDerived>& matrix,
+                                                                              const InverseType& trPerm) {
+    return Product<OtherDerived, InverseType, AliasFreeProduct>(matrix.derived(), trPerm.derived());
+  }
+
+  /** \returns the matrix with the inverse permutation applied to the rows.
+   */
+  template <typename OtherDerived>
+  const Product<InverseType, OtherDerived, AliasFreeProduct> operator*(const MatrixBase<OtherDerived>& matrix) const {
+    return Product<InverseType, OtherDerived, AliasFreeProduct>(derived(), matrix.derived());
+  }
+};
+
+template <typename Derived>
+const PermutationWrapper<const Derived> MatrixBase<Derived>::asPermutation() const {
+  return derived();
+}
+
+namespace internal {
+
+template <>
+struct AssignmentKind<DenseShape, PermutationShape> {
+  typedef EigenBase2EigenBase Kind;
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_PERMUTATIONMATRIX_H

include/Eigen/src/Core/Product.h

@@ -0,0 +1,307 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2011 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_PRODUCT_H
+#define EIGEN_PRODUCT_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+template <typename Lhs, typename Rhs, int Option, typename StorageKind>
+class ProductImpl;
+
+namespace internal {
+
+template <typename Lhs, typename Rhs, int Option>
+struct traits<Product<Lhs, Rhs, Option>> {
+  typedef remove_all_t<Lhs> LhsCleaned;
+  typedef remove_all_t<Rhs> RhsCleaned;
+  typedef traits<LhsCleaned> LhsTraits;
+  typedef traits<RhsCleaned> RhsTraits;
+
+  typedef MatrixXpr XprKind;
+
+  typedef typename ScalarBinaryOpTraits<typename traits<LhsCleaned>::Scalar,
+                                        typename traits<RhsCleaned>::Scalar>::ReturnType Scalar;
+  typedef typename product_promote_storage_type<typename LhsTraits::StorageKind, typename RhsTraits::StorageKind,
+                                                internal::product_type<Lhs, Rhs>::ret>::ret StorageKind;
+  typedef typename promote_index_type<typename LhsTraits::StorageIndex, typename RhsTraits::StorageIndex>::type
+      StorageIndex;
+
+  enum {
+    RowsAtCompileTime = LhsTraits::RowsAtCompileTime,
+    ColsAtCompileTime = RhsTraits::ColsAtCompileTime,
+    MaxRowsAtCompileTime = LhsTraits::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = RhsTraits::MaxColsAtCompileTime,
+
+    // FIXME: only needed by GeneralMatrixMatrixTriangular
+    InnerSize = min_size_prefer_fixed(LhsTraits::ColsAtCompileTime, RhsTraits::RowsAtCompileTime),
+
+    // The storage order is somewhat arbitrary here. The correct one will be determined through the evaluator.
+    Flags = (MaxRowsAtCompileTime == 1 && MaxColsAtCompileTime != 1)   ? RowMajorBit
+            : (MaxColsAtCompileTime == 1 && MaxRowsAtCompileTime != 1) ? 0
+            : (((LhsTraits::Flags & NoPreferredStorageOrderBit) && (RhsTraits::Flags & RowMajorBit)) ||
+               ((RhsTraits::Flags & NoPreferredStorageOrderBit) && (LhsTraits::Flags & RowMajorBit)))
+                ? RowMajorBit
+                : NoPreferredStorageOrderBit
+  };
+};
+
+struct TransposeProductEnum {
+  // convenience enumerations to specialize transposed products
+  enum : int {
+    Default = 0x00,
+    Matrix = 0x01,
+    Permutation = 0x02,
+    MatrixMatrix = (Matrix << 8) | Matrix,
+    MatrixPermutation = (Matrix << 8) | Permutation,
+    PermutationMatrix = (Permutation << 8) | Matrix
+  };
+};
+template <typename Xpr>
+struct TransposeKind {
+  static constexpr int Kind = is_matrix_base_xpr<Xpr>::value        ? TransposeProductEnum::Matrix
+                              : is_permutation_base_xpr<Xpr>::value ? TransposeProductEnum::Permutation
+                                                                    : TransposeProductEnum::Default;
+};
+
+template <typename Lhs, typename Rhs>
+struct TransposeProductKind {
+  static constexpr int Kind = (TransposeKind<Lhs>::Kind << 8) | TransposeKind<Rhs>::Kind;
+};
+
+template <typename Lhs, typename Rhs, int Option, int Kind = TransposeProductKind<Lhs, Rhs>::Kind>
+struct product_transpose_helper {
+  // by default, don't optimize the transposed product
+  using Derived = Product<Lhs, Rhs, Option>;
+  using Scalar = typename Derived::Scalar;
+  using TransposeType = Transpose<const Derived>;
+  using ConjugateTransposeType = CwiseUnaryOp<scalar_conjugate_op<Scalar>, TransposeType>;
+  using AdjointType = std::conditional_t<NumTraits<Scalar>::IsComplex, ConjugateTransposeType, TransposeType>;
+
+  // return (lhs * rhs)^T
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TransposeType run_transpose(const Derived& derived) {
+    return TransposeType(derived);
+  }
+  // return (lhs * rhs)^H
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AdjointType run_adjoint(const Derived& derived) {
+    return AdjointType(TransposeType(derived));
+  }
+};
+
+template <typename Lhs, typename Rhs, int Option>
+struct product_transpose_helper<Lhs, Rhs, Option, TransposeProductEnum::MatrixMatrix> {
+  // expand the transposed matrix-matrix product
+  using Derived = Product<Lhs, Rhs, Option>;
+
+  using LhsScalar = typename traits<Lhs>::Scalar;
+  using LhsTransposeType = typename DenseBase<Lhs>::ConstTransposeReturnType;
+  using LhsConjugateTransposeType = CwiseUnaryOp<scalar_conjugate_op<LhsScalar>, LhsTransposeType>;
+  using LhsAdjointType =
+      std::conditional_t<NumTraits<LhsScalar>::IsComplex, LhsConjugateTransposeType, LhsTransposeType>;
+
+  using RhsScalar = typename traits<Rhs>::Scalar;
+  using RhsTransposeType = typename DenseBase<Rhs>::ConstTransposeReturnType;
+  using RhsConjugateTransposeType = CwiseUnaryOp<scalar_conjugate_op<RhsScalar>, RhsTransposeType>;
+  using RhsAdjointType =
+      std::conditional_t<NumTraits<RhsScalar>::IsComplex, RhsConjugateTransposeType, RhsTransposeType>;
+
+  using TransposeType = Product<RhsTransposeType, LhsTransposeType, Option>;
+  using AdjointType = Product<RhsAdjointType, LhsAdjointType, Option>;
+
+  // return rhs^T * lhs^T
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TransposeType run_transpose(const Derived& derived) {
+    return TransposeType(RhsTransposeType(derived.rhs()), LhsTransposeType(derived.lhs()));
+  }
+  // return rhs^H * lhs^H
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AdjointType run_adjoint(const Derived& derived) {
+    return AdjointType(RhsAdjointType(RhsTransposeType(derived.rhs())),
+                       LhsAdjointType(LhsTransposeType(derived.lhs())));
+  }
+};
+template <typename Lhs, typename Rhs, int Option>
+struct product_transpose_helper<Lhs, Rhs, Option, TransposeProductEnum::PermutationMatrix> {
+  // expand the transposed permutation-matrix product
+  using Derived = Product<Lhs, Rhs, Option>;
+
+  using LhsInverseType = typename PermutationBase<Lhs>::InverseReturnType;
+
+  using RhsScalar = typename traits<Rhs>::Scalar;
+  using RhsTransposeType = typename DenseBase<Rhs>::ConstTransposeReturnType;
+  using RhsConjugateTransposeType = CwiseUnaryOp<scalar_conjugate_op<RhsScalar>, RhsTransposeType>;
+  using RhsAdjointType =
+      std::conditional_t<NumTraits<RhsScalar>::IsComplex, RhsConjugateTransposeType, RhsTransposeType>;
+
+  using TransposeType = Product<RhsTransposeType, LhsInverseType, Option>;
+  using AdjointType = Product<RhsAdjointType, LhsInverseType, Option>;
+
+  // return rhs^T * lhs^-1
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TransposeType run_transpose(const Derived& derived) {
+    return TransposeType(RhsTransposeType(derived.rhs()), LhsInverseType(derived.lhs()));
+  }
+  // return rhs^H * lhs^-1
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AdjointType run_adjoint(const Derived& derived) {
+    return AdjointType(RhsAdjointType(RhsTransposeType(derived.rhs())), LhsInverseType(derived.lhs()));
+  }
+};
+template <typename Lhs, typename Rhs, int Option>
+struct product_transpose_helper<Lhs, Rhs, Option, TransposeProductEnum::MatrixPermutation> {
+  // expand the transposed matrix-permutation product
+  using Derived = Product<Lhs, Rhs, Option>;
+
+  using LhsScalar = typename traits<Lhs>::Scalar;
+  using LhsTransposeType = typename DenseBase<Lhs>::ConstTransposeReturnType;
+  using LhsConjugateTransposeType = CwiseUnaryOp<scalar_conjugate_op<LhsScalar>, LhsTransposeType>;
+  using LhsAdjointType =
+      std::conditional_t<NumTraits<LhsScalar>::IsComplex, LhsConjugateTransposeType, LhsTransposeType>;
+
+  using RhsInverseType = typename PermutationBase<Rhs>::InverseReturnType;
+
+  using TransposeType = Product<RhsInverseType, LhsTransposeType, Option>;
+  using AdjointType = Product<RhsInverseType, LhsAdjointType, Option>;
+
+  // return rhs^-1 * lhs^T
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TransposeType run_transpose(const Derived& derived) {
+    return TransposeType(RhsInverseType(derived.rhs()), LhsTransposeType(derived.lhs()));
+  }
+  // return rhs^-1 * lhs^H
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AdjointType run_adjoint(const Derived& derived) {
+    return AdjointType(RhsInverseType(derived.rhs()), LhsAdjointType(LhsTransposeType(derived.lhs())));
+  }
+};
+
+}  // end namespace internal
+
+/** \class Product
+ * \ingroup Core_Module
+ *
+ * \brief Expression of the product of two arbitrary matrices or vectors
+ *
+ * \tparam Lhs_ the type of the left-hand side expression
+ * \tparam Rhs_ the type of the right-hand side expression
+ *
+ * This class represents an expression of the product of two arbitrary matrices.
+ *
+ * The other template parameters are:
+ * \tparam Option     can be DefaultProduct, AliasFreeProduct, or LazyProduct
+ *
+ */
+template <typename Lhs_, typename Rhs_, int Option>
+class Product
+    : public ProductImpl<Lhs_, Rhs_, Option,
+                         typename internal::product_promote_storage_type<
+                             typename internal::traits<Lhs_>::StorageKind, typename internal::traits<Rhs_>::StorageKind,
+                             internal::product_type<Lhs_, Rhs_>::ret>::ret> {
+ public:
+  typedef Lhs_ Lhs;
+  typedef Rhs_ Rhs;
+
+  typedef
+      typename ProductImpl<Lhs, Rhs, Option,
+                           typename internal::product_promote_storage_type<
+                               typename internal::traits<Lhs>::StorageKind, typename internal::traits<Rhs>::StorageKind,
+                               internal::product_type<Lhs, Rhs>::ret>::ret>::Base Base;
+  EIGEN_GENERIC_PUBLIC_INTERFACE(Product)
+
+  typedef typename internal::ref_selector<Lhs>::type LhsNested;
+  typedef typename internal::ref_selector<Rhs>::type RhsNested;
+  typedef internal::remove_all_t<LhsNested> LhsNestedCleaned;
+  typedef internal::remove_all_t<RhsNested> RhsNestedCleaned;
+
+  using TransposeReturnType = typename internal::product_transpose_helper<Lhs, Rhs, Option>::TransposeType;
+  using AdjointReturnType = typename internal::product_transpose_helper<Lhs, Rhs, Option>::AdjointType;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Product(const Lhs& lhs, const Rhs& rhs) : m_lhs(lhs), m_rhs(rhs) {
+    eigen_assert(lhs.cols() == rhs.rows() && "invalid matrix product" &&
+                 "if you wanted a coeff-wise or a dot product use the respective explicit functions");
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_lhs.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_rhs.cols(); }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const LhsNestedCleaned& lhs() const { return m_lhs; }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const RhsNestedCleaned& rhs() const { return m_rhs; }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TransposeReturnType transpose() const {
+    return internal::product_transpose_helper<Lhs, Rhs, Option>::run_transpose(*this);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AdjointReturnType adjoint() const {
+    return internal::product_transpose_helper<Lhs, Rhs, Option>::run_adjoint(*this);
+  }
+
+ protected:
+  LhsNested m_lhs;
+  RhsNested m_rhs;
+};
+
+namespace internal {
+
+template <typename Lhs, typename Rhs, int Option, int ProductTag = internal::product_type<Lhs, Rhs>::ret>
+class dense_product_base : public internal::dense_xpr_base<Product<Lhs, Rhs, Option>>::type {};
+
+/** Conversion to scalar for inner-products */
+template <typename Lhs, typename Rhs, int Option>
+class dense_product_base<Lhs, Rhs, Option, InnerProduct>
+    : public internal::dense_xpr_base<Product<Lhs, Rhs, Option>>::type {
+  typedef Product<Lhs, Rhs, Option> ProductXpr;
+  typedef typename internal::dense_xpr_base<ProductXpr>::type Base;
+
+ public:
+  using Base::derived;
+  typedef typename Base::Scalar Scalar;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE operator const Scalar() const {
+    return internal::evaluator<ProductXpr>(derived()).coeff(0, 0);
+  }
+};
+
+}  // namespace internal
+
+// Generic API dispatcher
+template <typename Lhs, typename Rhs, int Option, typename StorageKind>
+class ProductImpl : public internal::generic_xpr_base<Product<Lhs, Rhs, Option>, MatrixXpr, StorageKind>::type {
+ public:
+  typedef typename internal::generic_xpr_base<Product<Lhs, Rhs, Option>, MatrixXpr, StorageKind>::type Base;
+};
+
+template <typename Lhs, typename Rhs, int Option>
+class ProductImpl<Lhs, Rhs, Option, Dense> : public internal::dense_product_base<Lhs, Rhs, Option> {
+  typedef Product<Lhs, Rhs, Option> Derived;
+
+ public:
+  typedef typename internal::dense_product_base<Lhs, Rhs, Option> Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Derived)
+ protected:
+  enum {
+    IsOneByOne = (RowsAtCompileTime == 1 || RowsAtCompileTime == Dynamic) &&
+                 (ColsAtCompileTime == 1 || ColsAtCompileTime == Dynamic),
+    EnableCoeff = IsOneByOne || Option == LazyProduct
+  };
+
+ public:
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index row, Index col) const {
+    EIGEN_STATIC_ASSERT(EnableCoeff, THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS);
+    eigen_assert((Option == LazyProduct) || (this->rows() == 1 && this->cols() == 1));
+
+    return internal::evaluator<Derived>(derived()).coeff(row, col);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index i) const {
+    EIGEN_STATIC_ASSERT(EnableCoeff, THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS);
+    eigen_assert((Option == LazyProduct) || (this->rows() == 1 && this->cols() == 1));
+
+    return internal::evaluator<Derived>(derived()).coeff(i);
+  }
+};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_PRODUCT_H

include/Eigen/src/Core/Random.h

@@ -0,0 +1,207 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_RANDOM_H
+#define EIGEN_RANDOM_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Scalar>
+struct scalar_random_op {
+  inline const Scalar operator()() const { return random<Scalar>(); }
+};
+
+template <typename Scalar>
+struct functor_traits<scalar_random_op<Scalar> > {
+  enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false, IsRepeatable = false };
+};
+
+}  // end namespace internal
+
+/** \returns a random matrix expression
+ *
+ * Numbers are uniformly spread through their whole definition range for integer types,
+ * and in the [-1:1] range for floating point scalar types.
+ *
+ * The parameters \a rows and \a cols are the number of rows and of columns of
+ * the returned matrix. Must be compatible with this MatrixBase type.
+ *
+ * \not_reentrant
+ *
+ * This variant is meant to be used for dynamic-size matrix types. For fixed-size types,
+ * it is redundant to pass \a rows and \a cols as arguments, so Random() should be used
+ * instead.
+ *
+ *
+ * Example: \include MatrixBase_random_int_int.cpp
+ * Output: \verbinclude MatrixBase_random_int_int.out
+ *
+ * This expression has the "evaluate before nesting" flag so that it will be evaluated into
+ * a temporary matrix whenever it is nested in a larger expression. This prevents unexpected
+ * behavior with expressions involving random matrices.
+ *
+ * See DenseBase::NullaryExpr(Index, const CustomNullaryOp&) for an example using C++11 random generators.
+ *
+ * \sa DenseBase::setRandom(), DenseBase::Random(Index), DenseBase::Random()
+ */
+template <typename Derived>
+inline const typename DenseBase<Derived>::RandomReturnType DenseBase<Derived>::Random(Index rows, Index cols) {
+  return NullaryExpr(rows, cols, internal::scalar_random_op<Scalar>());
+}
+
+/** \returns a random vector expression
+ *
+ * Numbers are uniformly spread through their whole definition range for integer types,
+ * and in the [-1:1] range for floating point scalar types.
+ *
+ * The parameter \a size is the size of the returned vector.
+ * Must be compatible with this MatrixBase type.
+ *
+ * \only_for_vectors
+ * \not_reentrant
+ *
+ * This variant is meant to be used for dynamic-size vector types. For fixed-size types,
+ * it is redundant to pass \a size as argument, so Random() should be used
+ * instead.
+ *
+ * Example: \include MatrixBase_random_int.cpp
+ * Output: \verbinclude MatrixBase_random_int.out
+ *
+ * This expression has the "evaluate before nesting" flag so that it will be evaluated into
+ * a temporary vector whenever it is nested in a larger expression. This prevents unexpected
+ * behavior with expressions involving random matrices.
+ *
+ * \sa DenseBase::setRandom(), DenseBase::Random(Index,Index), DenseBase::Random()
+ */
+template <typename Derived>
+inline const typename DenseBase<Derived>::RandomReturnType DenseBase<Derived>::Random(Index size) {
+  return NullaryExpr(size, internal::scalar_random_op<Scalar>());
+}
+
+/** \returns a fixed-size random matrix or vector expression
+ *
+ * Numbers are uniformly spread through their whole definition range for integer types,
+ * and in the [-1:1] range for floating point scalar types.
+ *
+ * This variant is only for fixed-size MatrixBase types. For dynamic-size types, you
+ * need to use the variants taking size arguments.
+ *
+ * Example: \include MatrixBase_random.cpp
+ * Output: \verbinclude MatrixBase_random.out
+ *
+ * This expression has the "evaluate before nesting" flag so that it will be evaluated into
+ * a temporary matrix whenever it is nested in a larger expression. This prevents unexpected
+ * behavior with expressions involving random matrices.
+ *
+ * \not_reentrant
+ *
+ * \sa DenseBase::setRandom(), DenseBase::Random(Index,Index), DenseBase::Random(Index)
+ */
+template <typename Derived>
+inline const typename DenseBase<Derived>::RandomReturnType DenseBase<Derived>::Random() {
+  return NullaryExpr(RowsAtCompileTime, ColsAtCompileTime, internal::scalar_random_op<Scalar>());
+}
+
+/** Sets all coefficients in this expression to random values.
+ *
+ * Numbers are uniformly spread through their whole definition range for integer types,
+ * and in the [-1:1] range for floating point scalar types.
+ *
+ * \not_reentrant
+ *
+ * Example: \include MatrixBase_setRandom.cpp
+ * Output: \verbinclude MatrixBase_setRandom.out
+ *
+ * \sa class CwiseNullaryOp, setRandom(Index), setRandom(Index,Index)
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline Derived& DenseBase<Derived>::setRandom() {
+  return *this = Random(rows(), cols());
+}
+
+/** Resizes to the given \a newSize, and sets all coefficients in this expression to random values.
+ *
+ * Numbers are uniformly spread through their whole definition range for integer types,
+ * and in the [-1:1] range for floating point scalar types.
+ *
+ * \only_for_vectors
+ * \not_reentrant
+ *
+ * Example: \include Matrix_setRandom_int.cpp
+ * Output: \verbinclude Matrix_setRandom_int.out
+ *
+ * \sa DenseBase::setRandom(), setRandom(Index,Index), class CwiseNullaryOp, DenseBase::Random()
+ */
+template <typename Derived>
+EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setRandom(Index newSize) {
+  resize(newSize);
+  return setRandom();
+}
+
+/** Resizes to the given size, and sets all coefficients in this expression to random values.
+ *
+ * Numbers are uniformly spread through their whole definition range for integer types,
+ * and in the [-1:1] range for floating point scalar types.
+ *
+ * \not_reentrant
+ *
+ * \param rows the new number of rows
+ * \param cols the new number of columns
+ *
+ * Example: \include Matrix_setRandom_int_int.cpp
+ * Output: \verbinclude Matrix_setRandom_int_int.out
+ *
+ * \sa DenseBase::setRandom(), setRandom(Index), class CwiseNullaryOp, DenseBase::Random()
+ */
+template <typename Derived>
+EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setRandom(Index rows, Index cols) {
+  resize(rows, cols);
+  return setRandom();
+}
+
+/** Resizes to the given size, changing only the number of columns, and sets all
+ * coefficients in this expression to random values. For the parameter of type
+ * NoChange_t, just pass the special value \c NoChange.
+ *
+ * Numbers are uniformly spread through their whole definition range for integer types,
+ * and in the [-1:1] range for floating point scalar types.
+ *
+ * \not_reentrant
+ *
+ * \sa DenseBase::setRandom(), setRandom(Index), setRandom(Index, NoChange_t), class CwiseNullaryOp, DenseBase::Random()
+ */
+template <typename Derived>
+EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setRandom(NoChange_t, Index cols) {
+  return setRandom(rows(), cols);
+}
+
+/** Resizes to the given size, changing only the number of rows, and sets all
+ * coefficients in this expression to random values. For the parameter of type
+ * NoChange_t, just pass the special value \c NoChange.
+ *
+ * Numbers are uniformly spread through their whole definition range for integer types,
+ * and in the [-1:1] range for floating point scalar types.
+ *
+ * \not_reentrant
+ *
+ * \sa DenseBase::setRandom(), setRandom(Index), setRandom(NoChange_t, Index), class CwiseNullaryOp, DenseBase::Random()
+ */
+template <typename Derived>
+EIGEN_STRONG_INLINE Derived& PlainObjectBase<Derived>::setRandom(Index rows, NoChange_t) {
+  return setRandom(rows, cols());
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_RANDOM_H

include/Eigen/src/Core/RandomImpl.h

@@ -0,0 +1,255 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2024 Charles Schlosser <cs.schlosser@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_RANDOM_IMPL_H
+#define EIGEN_RANDOM_IMPL_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+/****************************************************************************
+ * Implementation of random                                               *
+ ****************************************************************************/
+
+template <typename Scalar, bool IsComplex, bool IsInteger>
+struct random_default_impl {};
+
+template <typename Scalar>
+struct random_impl : random_default_impl<Scalar, NumTraits<Scalar>::IsComplex, NumTraits<Scalar>::IsInteger> {};
+
+template <typename Scalar>
+struct random_retval {
+  typedef Scalar type;
+};
+
+template <typename Scalar>
+inline EIGEN_MATHFUNC_RETVAL(random, Scalar) random(const Scalar& x, const Scalar& y) {
+  return EIGEN_MATHFUNC_IMPL(random, Scalar)::run(x, y);
+}
+
+template <typename Scalar>
+inline EIGEN_MATHFUNC_RETVAL(random, Scalar) random() {
+  return EIGEN_MATHFUNC_IMPL(random, Scalar)::run();
+}
+
+// TODO: replace or provide alternatives to this, e.g. std::random_device
+struct eigen_random_device {
+  using ReturnType = int;
+  static constexpr int Entropy = meta_floor_log2<(unsigned int)(RAND_MAX) + 1>::value;
+  static constexpr ReturnType Highest = RAND_MAX;
+  static EIGEN_DEVICE_FUNC inline ReturnType run() { return std::rand(); }
+};
+
+// Fill a built-in unsigned integer with numRandomBits beginning with the least significant bit
+template <typename Scalar>
+struct random_bits_impl {
+  EIGEN_STATIC_ASSERT(std::is_unsigned<Scalar>::value, SCALAR MUST BE A BUILT - IN UNSIGNED INTEGER)
+  using RandomDevice = eigen_random_device;
+  using RandomReturnType = typename RandomDevice::ReturnType;
+  static constexpr int kEntropy = RandomDevice::Entropy;
+  static constexpr int kTotalBits = sizeof(Scalar) * CHAR_BIT;
+  // return a Scalar filled with numRandomBits beginning from the least significant bit
+  static EIGEN_DEVICE_FUNC inline Scalar run(int numRandomBits) {
+    eigen_assert((numRandomBits >= 0) && (numRandomBits <= kTotalBits));
+    const Scalar mask = Scalar(-1) >> ((kTotalBits - numRandomBits) & (kTotalBits - 1));
+    Scalar randomBits = 0;
+    for (int shift = 0; shift < numRandomBits; shift += kEntropy) {
+      RandomReturnType r = RandomDevice::run();
+      randomBits |= static_cast<Scalar>(r) << shift;
+    }
+    // clear the excess bits
+    randomBits &= mask;
+    return randomBits;
+  }
+};
+
+template <typename BitsType>
+EIGEN_DEVICE_FUNC inline BitsType getRandomBits(int numRandomBits) {
+  return random_bits_impl<BitsType>::run(numRandomBits);
+}
+
+// random implementation for a built-in floating point type
+template <typename Scalar, bool BuiltIn = std::is_floating_point<Scalar>::value>
+struct random_float_impl {
+  using BitsType = typename numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type;
+  static constexpr EIGEN_DEVICE_FUNC inline int mantissaBits() {
+    const int digits = NumTraits<Scalar>::digits();
+    return digits - 1;
+  }
+  static EIGEN_DEVICE_FUNC inline Scalar run(int numRandomBits) {
+    eigen_assert(numRandomBits >= 0 && numRandomBits <= mantissaBits());
+    BitsType randomBits = getRandomBits<BitsType>(numRandomBits);
+    // if fewer than MantissaBits is requested, shift them to the left
+    randomBits <<= (mantissaBits() - numRandomBits);
+    // randomBits is in the half-open interval [2,4)
+    randomBits |= numext::bit_cast<BitsType>(Scalar(2));
+    // result is in the half-open interval [-1,1)
+    Scalar result = numext::bit_cast<Scalar>(randomBits) - Scalar(3);
+    return result;
+  }
+};
+// random implementation for a custom floating point type
+// uses double as the implementation with a mantissa with a size equal to either the target scalar's mantissa or that of
+// double, whichever is smaller
+template <typename Scalar>
+struct random_float_impl<Scalar, false> {
+  static EIGEN_DEVICE_FUNC inline int mantissaBits() {
+    const int digits = NumTraits<Scalar>::digits();
+    constexpr int kDoubleDigits = NumTraits<double>::digits();
+    return numext::mini(digits, kDoubleDigits) - 1;
+  }
+  static EIGEN_DEVICE_FUNC inline Scalar run(int numRandomBits) {
+    eigen_assert(numRandomBits >= 0 && numRandomBits <= mantissaBits());
+    Scalar result = static_cast<Scalar>(random_float_impl<double>::run(numRandomBits));
+    return result;
+  }
+};
+
+#if !EIGEN_COMP_NVCC
+// random implementation for long double
+// this specialization is not compatible with double-double scalars
+template <bool Specialize = (sizeof(long double) == 2 * sizeof(uint64_t)) &&
+                            ((std::numeric_limits<long double>::digits != (2 * std::numeric_limits<double>::digits)))>
+struct random_longdouble_impl {
+  static constexpr int Size = sizeof(long double);
+  static constexpr EIGEN_DEVICE_FUNC inline int mantissaBits() { return NumTraits<long double>::digits() - 1; }
+  static EIGEN_DEVICE_FUNC inline long double run(int numRandomBits) {
+    eigen_assert(numRandomBits >= 0 && numRandomBits <= mantissaBits());
+    EIGEN_USING_STD(memcpy);
+    int numLowBits = numext::mini(numRandomBits, 64);
+    int numHighBits = numext::maxi(numRandomBits - 64, 0);
+    uint64_t randomBits[2];
+    long double result = 2.0L;
+    memcpy(&randomBits, &result, Size);
+    randomBits[0] |= getRandomBits<uint64_t>(numLowBits);
+    randomBits[1] |= getRandomBits<uint64_t>(numHighBits);
+    memcpy(&result, &randomBits, Size);
+    result -= 3.0L;
+    return result;
+  }
+};
+template <>
+struct random_longdouble_impl<false> {
+  static constexpr EIGEN_DEVICE_FUNC inline int mantissaBits() { return NumTraits<double>::digits() - 1; }
+  static EIGEN_DEVICE_FUNC inline long double run(int numRandomBits) {
+    return static_cast<long double>(random_float_impl<double>::run(numRandomBits));
+  }
+};
+template <>
+struct random_float_impl<long double> : random_longdouble_impl<> {};
+#endif
+
+template <typename Scalar>
+struct random_default_impl<Scalar, false, false> {
+  using Impl = random_float_impl<Scalar>;
+  static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y, int numRandomBits) {
+    Scalar half_x = Scalar(0.5) * x;
+    Scalar half_y = Scalar(0.5) * y;
+    Scalar result = (half_x + half_y) + (half_y - half_x) * run(numRandomBits);
+    // result is in the half-open interval [x, y) -- provided that x < y
+    return result;
+  }
+  static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y) {
+    return run(x, y, Impl::mantissaBits());
+  }
+  static EIGEN_DEVICE_FUNC inline Scalar run(int numRandomBits) { return Impl::run(numRandomBits); }
+  static EIGEN_DEVICE_FUNC inline Scalar run() { return run(Impl::mantissaBits()); }
+};
+
+template <typename Scalar, bool IsSigned = NumTraits<Scalar>::IsSigned, bool BuiltIn = std::is_integral<Scalar>::value>
+struct random_int_impl;
+
+// random implementation for a built-in unsigned integer type
+template <typename Scalar>
+struct random_int_impl<Scalar, false, true> {
+  static constexpr int kTotalBits = sizeof(Scalar) * CHAR_BIT;
+  static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y) {
+    if (y <= x) return x;
+    Scalar range = y - x;
+    // handle edge case where [x,y] spans the entire range of Scalar
+    if (range == NumTraits<Scalar>::highest()) return run();
+    Scalar count = range + 1;
+    // calculate the number of random bits needed to fill range
+    int numRandomBits = log2_ceil(count);
+    Scalar randomBits;
+    do {
+      randomBits = getRandomBits<Scalar>(numRandomBits);
+      // if the random draw is outside [0, range), try again (rejection sampling)
+      // in the worst-case scenario, the probability of rejection is: 1/2 - 1/2^numRandomBits < 50%
+    } while (randomBits >= count);
+    Scalar result = x + randomBits;
+    return result;
+  }
+  static EIGEN_DEVICE_FUNC inline Scalar run() { return getRandomBits<Scalar>(kTotalBits); }
+};
+
+// random implementation for a built-in signed integer type
+template <typename Scalar>
+struct random_int_impl<Scalar, true, true> {
+  static constexpr int kTotalBits = sizeof(Scalar) * CHAR_BIT;
+  using BitsType = typename make_unsigned<Scalar>::type;
+  static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y) {
+    if (y <= x) return x;
+    // Avoid overflow by representing `range` as an unsigned type
+    BitsType range = static_cast<BitsType>(y) - static_cast<BitsType>(x);
+    BitsType randomBits = random_int_impl<BitsType>::run(0, range);
+    // Avoid overflow in the case where `x` is negative and there is a large range so
+    // `randomBits` would also be negative if cast to `Scalar` first.
+    Scalar result = static_cast<Scalar>(static_cast<BitsType>(x) + randomBits);
+    return result;
+  }
+  static EIGEN_DEVICE_FUNC inline Scalar run() { return static_cast<Scalar>(getRandomBits<BitsType>(kTotalBits)); }
+};
+
+// todo: custom integers
+template <typename Scalar, bool IsSigned>
+struct random_int_impl<Scalar, IsSigned, false> {
+  static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar&, const Scalar&) { return run(); }
+  static EIGEN_DEVICE_FUNC inline Scalar run() {
+    eigen_assert(std::false_type::value && "RANDOM FOR CUSTOM INTEGERS NOT YET SUPPORTED");
+    return Scalar(0);
+  }
+};
+
+template <typename Scalar>
+struct random_default_impl<Scalar, false, true> : random_int_impl<Scalar> {};
+
+template <>
+struct random_impl<bool> {
+  static EIGEN_DEVICE_FUNC inline bool run(const bool& x, const bool& y) {
+    if (y <= x) return x;
+    return run();
+  }
+  static EIGEN_DEVICE_FUNC inline bool run() { return getRandomBits<unsigned>(1) ? true : false; }
+};
+
+template <typename Scalar>
+struct random_default_impl<Scalar, true, false> {
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+  using Impl = random_impl<RealScalar>;
+  static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y, int numRandomBits) {
+    return Scalar(Impl::run(x.real(), y.real(), numRandomBits), Impl::run(x.imag(), y.imag(), numRandomBits));
+  }
+  static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y) {
+    return Scalar(Impl::run(x.real(), y.real()), Impl::run(x.imag(), y.imag()));
+  }
+  static EIGEN_DEVICE_FUNC inline Scalar run(int numRandomBits) {
+    return Scalar(Impl::run(numRandomBits), Impl::run(numRandomBits));
+  }
+  static EIGEN_DEVICE_FUNC inline Scalar run() { return Scalar(Impl::run(), Impl::run()); }
+};
+
+}  // namespace internal
+}  // namespace Eigen
+
+#endif  // EIGEN_RANDOM_IMPL_H

include/Eigen/src/Core/Redux.h

@@ -0,0 +1,528 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_REDUX_H
+#define EIGEN_REDUX_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+// TODO
+//  * implement other kind of vectorization
+//  * factorize code
+
+/***************************************************************************
+ * Part 1 : the logic deciding a strategy for vectorization and unrolling
+ ***************************************************************************/
+
+template <typename Func, typename Evaluator>
+struct redux_traits {
+ public:
+  typedef typename find_best_packet<typename Evaluator::Scalar, Evaluator::SizeAtCompileTime>::type PacketType;
+  enum {
+    PacketSize = unpacket_traits<PacketType>::size,
+    InnerMaxSize = int(Evaluator::IsRowMajor) ? Evaluator::MaxColsAtCompileTime : Evaluator::MaxRowsAtCompileTime,
+    OuterMaxSize = int(Evaluator::IsRowMajor) ? Evaluator::MaxRowsAtCompileTime : Evaluator::MaxColsAtCompileTime,
+    SliceVectorizedWork = int(InnerMaxSize) == Dynamic   ? Dynamic
+                          : int(OuterMaxSize) == Dynamic ? (int(InnerMaxSize) >= int(PacketSize) ? Dynamic : 0)
+                                                         : (int(InnerMaxSize) / int(PacketSize)) * int(OuterMaxSize)
+  };
+
+  enum {
+    MayLinearize = (int(Evaluator::Flags) & LinearAccessBit),
+    MightVectorize = (int(Evaluator::Flags) & ActualPacketAccessBit) && (functor_traits<Func>::PacketAccess),
+    MayLinearVectorize = bool(MightVectorize) && bool(MayLinearize),
+    MaySliceVectorize = bool(MightVectorize) && (int(SliceVectorizedWork) == Dynamic || int(SliceVectorizedWork) >= 3)
+  };
+
+ public:
+  enum {
+    Traversal = int(MayLinearVectorize)  ? int(LinearVectorizedTraversal)
+                : int(MaySliceVectorize) ? int(SliceVectorizedTraversal)
+                : int(MayLinearize)      ? int(LinearTraversal)
+                                         : int(DefaultTraversal)
+  };
+
+ public:
+  enum {
+    Cost = Evaluator::SizeAtCompileTime == Dynamic
+               ? HugeCost
+               : int(Evaluator::SizeAtCompileTime) * int(Evaluator::CoeffReadCost) +
+                     (Evaluator::SizeAtCompileTime - 1) * functor_traits<Func>::Cost,
+    UnrollingLimit = EIGEN_UNROLLING_LIMIT * (int(Traversal) == int(DefaultTraversal) ? 1 : int(PacketSize))
+  };
+
+ public:
+  enum { Unrolling = Cost <= UnrollingLimit ? CompleteUnrolling : NoUnrolling };
+
+#ifdef EIGEN_DEBUG_ASSIGN
+  static void debug() {
+    std::cerr << "Xpr: " << typeid(typename Evaluator::XprType).name() << std::endl;
+    std::cerr.setf(std::ios::hex, std::ios::basefield);
+    EIGEN_DEBUG_VAR(Evaluator::Flags)
+    std::cerr.unsetf(std::ios::hex);
+    EIGEN_DEBUG_VAR(InnerMaxSize)
+    EIGEN_DEBUG_VAR(OuterMaxSize)
+    EIGEN_DEBUG_VAR(SliceVectorizedWork)
+    EIGEN_DEBUG_VAR(PacketSize)
+    EIGEN_DEBUG_VAR(MightVectorize)
+    EIGEN_DEBUG_VAR(MayLinearVectorize)
+    EIGEN_DEBUG_VAR(MaySliceVectorize)
+    std::cerr << "Traversal"
+              << " = " << Traversal << " (" << demangle_traversal(Traversal) << ")" << std::endl;
+    EIGEN_DEBUG_VAR(UnrollingLimit)
+    std::cerr << "Unrolling"
+              << " = " << Unrolling << " (" << demangle_unrolling(Unrolling) << ")" << std::endl;
+    std::cerr << std::endl;
+  }
+#endif
+};
+
+/***************************************************************************
+ * Part 2 : unrollers
+ ***************************************************************************/
+
+/*** no vectorization ***/
+
+template <typename Func, typename Evaluator, Index Start, Index Length>
+struct redux_novec_unroller {
+  static constexpr Index HalfLength = Length / 2;
+
+  typedef typename Evaluator::Scalar Scalar;
+
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval, const Func& func) {
+    return func(redux_novec_unroller<Func, Evaluator, Start, HalfLength>::run(eval, func),
+                redux_novec_unroller<Func, Evaluator, Start + HalfLength, Length - HalfLength>::run(eval, func));
+  }
+};
+
+template <typename Func, typename Evaluator, Index Start>
+struct redux_novec_unroller<Func, Evaluator, Start, 1> {
+  static constexpr Index outer = Start / Evaluator::InnerSizeAtCompileTime;
+  static constexpr Index inner = Start % Evaluator::InnerSizeAtCompileTime;
+
+  typedef typename Evaluator::Scalar Scalar;
+
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval, const Func&) {
+    return eval.coeffByOuterInner(outer, inner);
+  }
+};
+
+// This is actually dead code and will never be called. It is required
+// to prevent false warnings regarding failed inlining though
+// for 0 length run() will never be called at all.
+template <typename Func, typename Evaluator, Index Start>
+struct redux_novec_unroller<Func, Evaluator, Start, 0> {
+  typedef typename Evaluator::Scalar Scalar;
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator&, const Func&) { return Scalar(); }
+};
+
+template <typename Func, typename Evaluator, Index Start, Index Length>
+struct redux_novec_linear_unroller {
+  static constexpr Index HalfLength = Length / 2;
+
+  typedef typename Evaluator::Scalar Scalar;
+
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval, const Func& func) {
+    return func(redux_novec_linear_unroller<Func, Evaluator, Start, HalfLength>::run(eval, func),
+                redux_novec_linear_unroller<Func, Evaluator, Start + HalfLength, Length - HalfLength>::run(eval, func));
+  }
+};
+
+template <typename Func, typename Evaluator, Index Start>
+struct redux_novec_linear_unroller<Func, Evaluator, Start, 1> {
+  typedef typename Evaluator::Scalar Scalar;
+
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval, const Func&) {
+    return eval.coeff(Start);
+  }
+};
+
+// This is actually dead code and will never be called. It is required
+// to prevent false warnings regarding failed inlining though
+// for 0 length run() will never be called at all.
+template <typename Func, typename Evaluator, Index Start>
+struct redux_novec_linear_unroller<Func, Evaluator, Start, 0> {
+  typedef typename Evaluator::Scalar Scalar;
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator&, const Func&) { return Scalar(); }
+};
+
+/*** vectorization ***/
+
+template <typename Func, typename Evaluator, Index Start, Index Length>
+struct redux_vec_unroller {
+  template <typename PacketType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator& eval, const Func& func) {
+    constexpr Index HalfLength = Length / 2;
+
+    return func.packetOp(
+        redux_vec_unroller<Func, Evaluator, Start, HalfLength>::template run<PacketType>(eval, func),
+        redux_vec_unroller<Func, Evaluator, Start + HalfLength, Length - HalfLength>::template run<PacketType>(eval,
+                                                                                                               func));
+  }
+};
+
+template <typename Func, typename Evaluator, Index Start>
+struct redux_vec_unroller<Func, Evaluator, Start, 1> {
+  template <typename PacketType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator& eval, const Func&) {
+    constexpr Index PacketSize = unpacket_traits<PacketType>::size;
+    constexpr Index index = Start * PacketSize;
+    constexpr Index outer = index / int(Evaluator::InnerSizeAtCompileTime);
+    constexpr Index inner = index % int(Evaluator::InnerSizeAtCompileTime);
+    constexpr int alignment = Evaluator::Alignment;
+
+    return eval.template packetByOuterInner<alignment, PacketType>(outer, inner);
+  }
+};
+
+template <typename Func, typename Evaluator, Index Start, Index Length>
+struct redux_vec_linear_unroller {
+  template <typename PacketType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator& eval, const Func& func) {
+    constexpr Index HalfLength = Length / 2;
+
+    return func.packetOp(
+        redux_vec_linear_unroller<Func, Evaluator, Start, HalfLength>::template run<PacketType>(eval, func),
+        redux_vec_linear_unroller<Func, Evaluator, Start + HalfLength, Length - HalfLength>::template run<PacketType>(
+            eval, func));
+  }
+};
+
+template <typename Func, typename Evaluator, Index Start>
+struct redux_vec_linear_unroller<Func, Evaluator, Start, 1> {
+  template <typename PacketType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE PacketType run(const Evaluator& eval, const Func&) {
+    constexpr Index PacketSize = unpacket_traits<PacketType>::size;
+    constexpr Index index = (Start * PacketSize);
+    constexpr int alignment = Evaluator::Alignment;
+    return eval.template packet<alignment, PacketType>(index);
+  }
+};
+
+/***************************************************************************
+ * Part 3 : implementation of all cases
+ ***************************************************************************/
+
+template <typename Func, typename Evaluator, int Traversal = redux_traits<Func, Evaluator>::Traversal,
+          int Unrolling = redux_traits<Func, Evaluator>::Unrolling>
+struct redux_impl;
+
+template <typename Func, typename Evaluator>
+struct redux_impl<Func, Evaluator, DefaultTraversal, NoUnrolling> {
+  typedef typename Evaluator::Scalar Scalar;
+
+  template <typename XprType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval, const Func& func, const XprType& xpr) {
+    eigen_assert(xpr.rows() > 0 && xpr.cols() > 0 && "you are using an empty matrix");
+    Scalar res = eval.coeffByOuterInner(0, 0);
+    for (Index i = 1; i < xpr.innerSize(); ++i) res = func(res, eval.coeffByOuterInner(0, i));
+    for (Index i = 1; i < xpr.outerSize(); ++i)
+      for (Index j = 0; j < xpr.innerSize(); ++j) res = func(res, eval.coeffByOuterInner(i, j));
+    return res;
+  }
+};
+
+template <typename Func, typename Evaluator>
+struct redux_impl<Func, Evaluator, LinearTraversal, NoUnrolling> {
+  typedef typename Evaluator::Scalar Scalar;
+
+  template <typename XprType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval, const Func& func, const XprType& xpr) {
+    eigen_assert(xpr.size() > 0 && "you are using an empty matrix");
+    Scalar res = eval.coeff(0);
+    for (Index k = 1; k < xpr.size(); ++k) res = func(res, eval.coeff(k));
+    return res;
+  }
+};
+
+template <typename Func, typename Evaluator>
+struct redux_impl<Func, Evaluator, DefaultTraversal, CompleteUnrolling>
+    : redux_novec_unroller<Func, Evaluator, 0, Evaluator::SizeAtCompileTime> {
+  typedef redux_novec_unroller<Func, Evaluator, 0, Evaluator::SizeAtCompileTime> Base;
+  typedef typename Evaluator::Scalar Scalar;
+  template <typename XprType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval, const Func& func,
+                                                          const XprType& /*xpr*/) {
+    return Base::run(eval, func);
+  }
+};
+
+template <typename Func, typename Evaluator>
+struct redux_impl<Func, Evaluator, LinearTraversal, CompleteUnrolling>
+    : redux_novec_linear_unroller<Func, Evaluator, 0, Evaluator::SizeAtCompileTime> {
+  typedef redux_novec_linear_unroller<Func, Evaluator, 0, Evaluator::SizeAtCompileTime> Base;
+  typedef typename Evaluator::Scalar Scalar;
+  template <typename XprType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval, const Func& func,
+                                                          const XprType& /*xpr*/) {
+    return Base::run(eval, func);
+  }
+};
+
+template <typename Func, typename Evaluator>
+struct redux_impl<Func, Evaluator, LinearVectorizedTraversal, NoUnrolling> {
+  typedef typename Evaluator::Scalar Scalar;
+  typedef typename redux_traits<Func, Evaluator>::PacketType PacketScalar;
+
+  template <typename XprType>
+  static Scalar run(const Evaluator& eval, const Func& func, const XprType& xpr) {
+    const Index size = xpr.size();
+
+    constexpr Index packetSize = redux_traits<Func, Evaluator>::PacketSize;
+    constexpr int packetAlignment = unpacket_traits<PacketScalar>::alignment;
+    constexpr int alignment0 =
+        (bool(Evaluator::Flags & DirectAccessBit) && bool(packet_traits<Scalar>::AlignedOnScalar))
+            ? int(packetAlignment)
+            : int(Unaligned);
+    constexpr int alignment = plain_enum_max(alignment0, Evaluator::Alignment);
+    const Index alignedStart = internal::first_default_aligned(xpr);
+    const Index alignedSize2 = ((size - alignedStart) / (2 * packetSize)) * (2 * packetSize);
+    const Index alignedSize = ((size - alignedStart) / (packetSize)) * (packetSize);
+    const Index alignedEnd2 = alignedStart + alignedSize2;
+    const Index alignedEnd = alignedStart + alignedSize;
+    Scalar res;
+    if (alignedSize) {
+      PacketScalar packet_res0 = eval.template packet<alignment, PacketScalar>(alignedStart);
+      if (alignedSize > packetSize)  // we have at least two packets to partly unroll the loop
+      {
+        PacketScalar packet_res1 = eval.template packet<alignment, PacketScalar>(alignedStart + packetSize);
+        for (Index index = alignedStart + 2 * packetSize; index < alignedEnd2; index += 2 * packetSize) {
+          packet_res0 = func.packetOp(packet_res0, eval.template packet<alignment, PacketScalar>(index));
+          packet_res1 = func.packetOp(packet_res1, eval.template packet<alignment, PacketScalar>(index + packetSize));
+        }
+
+        packet_res0 = func.packetOp(packet_res0, packet_res1);
+        if (alignedEnd > alignedEnd2)
+          packet_res0 = func.packetOp(packet_res0, eval.template packet<alignment, PacketScalar>(alignedEnd2));
+      }
+      res = func.predux(packet_res0);
+
+      for (Index index = 0; index < alignedStart; ++index) res = func(res, eval.coeff(index));
+
+      for (Index index = alignedEnd; index < size; ++index) res = func(res, eval.coeff(index));
+    } else  // too small to vectorize anything.
+            // since this is dynamic-size hence inefficient anyway for such small sizes, don't try to optimize.
+    {
+      res = eval.coeff(0);
+      for (Index index = 1; index < size; ++index) res = func(res, eval.coeff(index));
+    }
+
+    return res;
+  }
+};
+
+// NOTE: for SliceVectorizedTraversal we simply bypass unrolling
+template <typename Func, typename Evaluator, int Unrolling>
+struct redux_impl<Func, Evaluator, SliceVectorizedTraversal, Unrolling> {
+  typedef typename Evaluator::Scalar Scalar;
+  typedef typename redux_traits<Func, Evaluator>::PacketType PacketType;
+
+  template <typename XprType>
+  EIGEN_DEVICE_FUNC static Scalar run(const Evaluator& eval, const Func& func, const XprType& xpr) {
+    eigen_assert(xpr.rows() > 0 && xpr.cols() > 0 && "you are using an empty matrix");
+    constexpr Index packetSize = redux_traits<Func, Evaluator>::PacketSize;
+    const Index innerSize = xpr.innerSize();
+    const Index outerSize = xpr.outerSize();
+    const Index packetedInnerSize = ((innerSize) / packetSize) * packetSize;
+    Scalar res;
+    if (packetedInnerSize) {
+      PacketType packet_res = eval.template packet<Unaligned, PacketType>(0, 0);
+      for (Index j = 0; j < outerSize; ++j)
+        for (Index i = (j == 0 ? packetSize : 0); i < packetedInnerSize; i += Index(packetSize))
+          packet_res = func.packetOp(packet_res, eval.template packetByOuterInner<Unaligned, PacketType>(j, i));
+
+      res = func.predux(packet_res);
+      for (Index j = 0; j < outerSize; ++j)
+        for (Index i = packetedInnerSize; i < innerSize; ++i) res = func(res, eval.coeffByOuterInner(j, i));
+    } else  // too small to vectorize anything.
+            // since this is dynamic-size hence inefficient anyway for such small sizes, don't try to optimize.
+    {
+      res = redux_impl<Func, Evaluator, DefaultTraversal, NoUnrolling>::run(eval, func, xpr);
+    }
+
+    return res;
+  }
+};
+
+template <typename Func, typename Evaluator>
+struct redux_impl<Func, Evaluator, LinearVectorizedTraversal, CompleteUnrolling> {
+  typedef typename Evaluator::Scalar Scalar;
+
+  typedef typename redux_traits<Func, Evaluator>::PacketType PacketType;
+  static constexpr Index PacketSize = redux_traits<Func, Evaluator>::PacketSize;
+  static constexpr Index Size = Evaluator::SizeAtCompileTime;
+  static constexpr Index VectorizedSize = (int(Size) / int(PacketSize)) * int(PacketSize);
+
+  template <typename XprType>
+  EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval, const Func& func, const XprType& xpr) {
+    EIGEN_ONLY_USED_FOR_DEBUG(xpr)
+    eigen_assert(xpr.rows() > 0 && xpr.cols() > 0 && "you are using an empty matrix");
+    if (VectorizedSize > 0) {
+      Scalar res = func.predux(
+          redux_vec_linear_unroller<Func, Evaluator, 0, Size / PacketSize>::template run<PacketType>(eval, func));
+      if (VectorizedSize != Size)
+        res = func(
+            res, redux_novec_linear_unroller<Func, Evaluator, VectorizedSize, Size - VectorizedSize>::run(eval, func));
+      return res;
+    } else {
+      return redux_novec_linear_unroller<Func, Evaluator, 0, Size>::run(eval, func);
+    }
+  }
+};
+
+// evaluator adaptor
+template <typename XprType_>
+class redux_evaluator : public internal::evaluator<XprType_> {
+  typedef internal::evaluator<XprType_> Base;
+
+ public:
+  typedef XprType_ XprType;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit redux_evaluator(const XprType& xpr) : Base(xpr) {}
+
+  typedef typename XprType::Scalar Scalar;
+  typedef typename XprType::CoeffReturnType CoeffReturnType;
+  typedef typename XprType::PacketScalar PacketScalar;
+
+  enum {
+    MaxRowsAtCompileTime = XprType::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = XprType::MaxColsAtCompileTime,
+    // TODO we should not remove DirectAccessBit and rather find an elegant way to query the alignment offset at runtime
+    // from the evaluator
+    Flags = Base::Flags & ~DirectAccessBit,
+    IsRowMajor = XprType::IsRowMajor,
+    SizeAtCompileTime = XprType::SizeAtCompileTime,
+    InnerSizeAtCompileTime = XprType::InnerSizeAtCompileTime
+  };
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeffByOuterInner(Index outer, Index inner) const {
+    return Base::coeff(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer);
+  }
+
+  template <int LoadMode, typename PacketType>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetByOuterInner(Index outer, Index inner) const {
+    return Base::template packet<LoadMode, PacketType>(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer);
+  }
+};
+
+}  // end namespace internal
+
+/***************************************************************************
+ * Part 4 : public API
+ ***************************************************************************/
+
+/** \returns the result of a full redux operation on the whole matrix or vector using \a func
+ *
+ * The template parameter \a BinaryOp is the type of the functor \a func which must be
+ * an associative operator. Both current C++98 and C++11 functor styles are handled.
+ *
+ * \warning the matrix must be not empty, otherwise an assertion is triggered.
+ *
+ * \sa DenseBase::sum(), DenseBase::minCoeff(), DenseBase::maxCoeff(), MatrixBase::colwise(), MatrixBase::rowwise()
+ */
+template <typename Derived>
+template <typename Func>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::redux(
+    const Func& func) const {
+  eigen_assert(this->rows() > 0 && this->cols() > 0 && "you are using an empty matrix");
+
+  typedef typename internal::redux_evaluator<Derived> ThisEvaluator;
+  ThisEvaluator thisEval(derived());
+
+  // The initial expression is passed to the reducer as an additional argument instead of
+  // passing it as a member of redux_evaluator to help
+  return internal::redux_impl<Func, ThisEvaluator>::run(thisEval, func, derived());
+}
+
+/** \returns the minimum of all coefficients of \c *this.
+ * In case \c *this contains NaN, NaNPropagation determines the behavior:
+ *   NaNPropagation == PropagateFast : undefined
+ *   NaNPropagation == PropagateNaN : result is NaN
+ *   NaNPropagation == PropagateNumbers : result is minimum of elements that are not NaN
+ * \warning the matrix must be not empty, otherwise an assertion is triggered.
+ */
+template <typename Derived>
+template <int NaNPropagation>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::minCoeff() const {
+  return derived().redux(Eigen::internal::scalar_min_op<Scalar, Scalar, NaNPropagation>());
+}
+
+/** \returns the maximum of all coefficients of \c *this.
+ * In case \c *this contains NaN, NaNPropagation determines the behavior:
+ *   NaNPropagation == PropagateFast : undefined
+ *   NaNPropagation == PropagateNaN : result is NaN
+ *   NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN
+ * \warning the matrix must be not empty, otherwise an assertion is triggered.
+ */
+template <typename Derived>
+template <int NaNPropagation>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::maxCoeff() const {
+  return derived().redux(Eigen::internal::scalar_max_op<Scalar, Scalar, NaNPropagation>());
+}
+
+/** \returns the sum of all coefficients of \c *this
+ *
+ * If \c *this is empty, then the value 0 is returned.
+ *
+ * \sa trace(), prod(), mean()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::sum() const {
+  if (SizeAtCompileTime == 0 || (SizeAtCompileTime == Dynamic && size() == 0)) return Scalar(0);
+  return derived().redux(Eigen::internal::scalar_sum_op<Scalar, Scalar>());
+}
+
+/** \returns the mean of all coefficients of *this
+ *
+ * \sa trace(), prod(), sum()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::mean() const {
+#ifdef __INTEL_COMPILER
+#pragma warning push
+#pragma warning(disable : 2259)
+#endif
+  return Scalar(derived().redux(Eigen::internal::scalar_sum_op<Scalar, Scalar>())) / Scalar(this->size());
+#ifdef __INTEL_COMPILER
+#pragma warning pop
+#endif
+}
+
+/** \returns the product of all coefficients of *this
+ *
+ * Example: \include MatrixBase_prod.cpp
+ * Output: \verbinclude MatrixBase_prod.out
+ *
+ * \sa sum(), mean(), trace()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::prod() const {
+  if (SizeAtCompileTime == 0 || (SizeAtCompileTime == Dynamic && size() == 0)) return Scalar(1);
+  return derived().redux(Eigen::internal::scalar_product_op<Scalar>());
+}
+
+/** \returns the trace of \c *this, i.e. the sum of the coefficients on the main diagonal.
+ *
+ * \c *this can be any matrix, not necessarily square.
+ *
+ * \sa diagonal(), sum()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar MatrixBase<Derived>::trace() const {
+  return derived().diagonal().sum();
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_REDUX_H

include/Eigen/src/Core/Ref.h

@@ -0,0 +1,383 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2012 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_REF_H
+#define EIGEN_REF_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename PlainObjectType_, int Options_, typename StrideType_>
+struct traits<Ref<PlainObjectType_, Options_, StrideType_> >
+    : public traits<Map<PlainObjectType_, Options_, StrideType_> > {
+  typedef PlainObjectType_ PlainObjectType;
+  typedef StrideType_ StrideType;
+  enum {
+    Options = Options_,
+    Flags = traits<Map<PlainObjectType_, Options_, StrideType_> >::Flags | NestByRefBit,
+    Alignment = traits<Map<PlainObjectType_, Options_, StrideType_> >::Alignment,
+    InnerStrideAtCompileTime = traits<Map<PlainObjectType_, Options_, StrideType_> >::InnerStrideAtCompileTime,
+    OuterStrideAtCompileTime = traits<Map<PlainObjectType_, Options_, StrideType_> >::OuterStrideAtCompileTime
+  };
+
+  template <typename Derived>
+  struct match {
+    enum {
+      IsVectorAtCompileTime = PlainObjectType::IsVectorAtCompileTime || Derived::IsVectorAtCompileTime,
+      HasDirectAccess = internal::has_direct_access<Derived>::ret,
+      StorageOrderMatch =
+          IsVectorAtCompileTime || ((PlainObjectType::Flags & RowMajorBit) == (Derived::Flags & RowMajorBit)),
+      InnerStrideMatch = int(InnerStrideAtCompileTime) == int(Dynamic) ||
+                         int(InnerStrideAtCompileTime) == int(Derived::InnerStrideAtCompileTime) ||
+                         (int(InnerStrideAtCompileTime) == 0 && int(Derived::InnerStrideAtCompileTime) == 1),
+      OuterStrideMatch = IsVectorAtCompileTime || int(OuterStrideAtCompileTime) == int(Dynamic) ||
+                         int(OuterStrideAtCompileTime) == int(Derived::OuterStrideAtCompileTime),
+      // NOTE, this indirection of evaluator<Derived>::Alignment is needed
+      // to workaround a very strange bug in MSVC related to the instantiation
+      // of has_*ary_operator in evaluator<CwiseNullaryOp>.
+      // This line is surprisingly very sensitive. For instance, simply adding parenthesis
+      // as "DerivedAlignment = (int(evaluator<Derived>::Alignment))," will make MSVC fail...
+      DerivedAlignment = int(evaluator<Derived>::Alignment),
+      AlignmentMatch = (int(traits<PlainObjectType>::Alignment) == int(Unaligned)) ||
+                       (DerivedAlignment >= int(Alignment)),  // FIXME the first condition is not very clear, it should
+                                                              // be replaced by the required alignment
+      ScalarTypeMatch = internal::is_same<typename PlainObjectType::Scalar, typename Derived::Scalar>::value,
+      MatchAtCompileTime = HasDirectAccess && StorageOrderMatch && InnerStrideMatch && OuterStrideMatch &&
+                           AlignmentMatch && ScalarTypeMatch
+    };
+    typedef std::conditional_t<MatchAtCompileTime, internal::true_type, internal::false_type> type;
+  };
+};
+
+template <typename Derived>
+struct traits<RefBase<Derived> > : public traits<Derived> {};
+
+}  // namespace internal
+
+template <typename Derived>
+class RefBase : public MapBase<Derived> {
+  typedef typename internal::traits<Derived>::PlainObjectType PlainObjectType;
+  typedef typename internal::traits<Derived>::StrideType StrideType;
+
+ public:
+  typedef MapBase<Derived> Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(RefBase)
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const {
+    return StrideType::InnerStrideAtCompileTime != 0 ? m_stride.inner() : 1;
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const {
+    return StrideType::OuterStrideAtCompileTime != 0 ? m_stride.outer()
+           : IsVectorAtCompileTime                   ? this->size()
+           : int(Flags) & RowMajorBit                ? this->cols()
+                                                     : this->rows();
+  }
+
+  EIGEN_DEVICE_FUNC RefBase()
+      : Base(0, RowsAtCompileTime == Dynamic ? 0 : RowsAtCompileTime,
+             ColsAtCompileTime == Dynamic ? 0 : ColsAtCompileTime),
+        // Stride<> does not allow default ctor for Dynamic strides, so let' initialize it with dummy values:
+        m_stride(StrideType::OuterStrideAtCompileTime == Dynamic ? 0 : StrideType::OuterStrideAtCompileTime,
+                 StrideType::InnerStrideAtCompileTime == Dynamic ? 0 : StrideType::InnerStrideAtCompileTime) {}
+
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(RefBase)
+
+ protected:
+  typedef Stride<StrideType::OuterStrideAtCompileTime, StrideType::InnerStrideAtCompileTime> StrideBase;
+
+  // Resolves inner stride if default 0.
+  static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index resolveInnerStride(Index inner) { return inner == 0 ? 1 : inner; }
+
+  // Resolves outer stride if default 0.
+  static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index resolveOuterStride(Index inner, Index outer, Index rows, Index cols,
+                                                                    bool isVectorAtCompileTime, bool isRowMajor) {
+    return outer == 0 ? isVectorAtCompileTime ? inner * rows * cols : isRowMajor ? inner * cols : inner * rows : outer;
+  }
+
+  // Returns true if construction is valid, false if there is a stride mismatch,
+  // and fails if there is a size mismatch.
+  template <typename Expression>
+  EIGEN_DEVICE_FUNC bool construct(Expression& expr) {
+    // Check matrix sizes.  If this is a compile-time vector, we do allow
+    // implicitly transposing.
+    EIGEN_STATIC_ASSERT(EIGEN_PREDICATE_SAME_MATRIX_SIZE(PlainObjectType, Expression)
+                            // If it is a vector, the transpose sizes might match.
+                            || (PlainObjectType::IsVectorAtCompileTime &&
+                                ((int(PlainObjectType::RowsAtCompileTime) == Eigen::Dynamic ||
+                                  int(Expression::ColsAtCompileTime) == Eigen::Dynamic ||
+                                  int(PlainObjectType::RowsAtCompileTime) == int(Expression::ColsAtCompileTime)) &&
+                                 (int(PlainObjectType::ColsAtCompileTime) == Eigen::Dynamic ||
+                                  int(Expression::RowsAtCompileTime) == Eigen::Dynamic ||
+                                  int(PlainObjectType::ColsAtCompileTime) == int(Expression::RowsAtCompileTime)))),
+                        YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES)
+
+    // Determine runtime rows and columns.
+    Index rows = expr.rows();
+    Index cols = expr.cols();
+    if (PlainObjectType::RowsAtCompileTime == 1) {
+      eigen_assert(expr.rows() == 1 || expr.cols() == 1);
+      rows = 1;
+      cols = expr.size();
+    } else if (PlainObjectType::ColsAtCompileTime == 1) {
+      eigen_assert(expr.rows() == 1 || expr.cols() == 1);
+      rows = expr.size();
+      cols = 1;
+    }
+    // Verify that the sizes are valid.
+    eigen_assert((PlainObjectType::RowsAtCompileTime == Dynamic) || (PlainObjectType::RowsAtCompileTime == rows));
+    eigen_assert((PlainObjectType::ColsAtCompileTime == Dynamic) || (PlainObjectType::ColsAtCompileTime == cols));
+
+    // If this is a vector, we might be transposing, which means that stride should swap.
+    const bool transpose = PlainObjectType::IsVectorAtCompileTime && (rows != expr.rows());
+    // If the storage format differs, we also need to swap the stride.
+    const bool row_major = ((PlainObjectType::Flags)&RowMajorBit) != 0;
+    const bool expr_row_major = (Expression::Flags & RowMajorBit) != 0;
+    const bool storage_differs = (row_major != expr_row_major);
+
+    const bool swap_stride = (transpose != storage_differs);
+
+    // Determine expr's actual strides, resolving any defaults if zero.
+    const Index expr_inner_actual = resolveInnerStride(expr.innerStride());
+    const Index expr_outer_actual = resolveOuterStride(expr_inner_actual, expr.outerStride(), expr.rows(), expr.cols(),
+                                                       Expression::IsVectorAtCompileTime != 0, expr_row_major);
+
+    // If this is a column-major row vector or row-major column vector, the inner-stride
+    // is arbitrary, so set it to either the compile-time inner stride or 1.
+    const bool row_vector = (rows == 1);
+    const bool col_vector = (cols == 1);
+    const Index inner_stride =
+        ((!row_major && row_vector) || (row_major && col_vector))
+            ? (StrideType::InnerStrideAtCompileTime > 0 ? Index(StrideType::InnerStrideAtCompileTime) : 1)
+        : swap_stride ? expr_outer_actual
+                      : expr_inner_actual;
+
+    // If this is a column-major column vector or row-major row vector, the outer-stride
+    // is arbitrary, so set it to either the compile-time outer stride or vector size.
+    const Index outer_stride =
+        ((!row_major && col_vector) || (row_major && row_vector))
+            ? (StrideType::OuterStrideAtCompileTime > 0 ? Index(StrideType::OuterStrideAtCompileTime)
+                                                        : rows * cols * inner_stride)
+        : swap_stride ? expr_inner_actual
+                      : expr_outer_actual;
+
+    // Check if given inner/outer strides are compatible with compile-time strides.
+    const bool inner_valid = (StrideType::InnerStrideAtCompileTime == Dynamic) ||
+                             (resolveInnerStride(Index(StrideType::InnerStrideAtCompileTime)) == inner_stride);
+    if (!inner_valid) {
+      return false;
+    }
+
+    const bool outer_valid =
+        (StrideType::OuterStrideAtCompileTime == Dynamic) ||
+        (resolveOuterStride(inner_stride, Index(StrideType::OuterStrideAtCompileTime), rows, cols,
+                            PlainObjectType::IsVectorAtCompileTime != 0, row_major) == outer_stride);
+    if (!outer_valid) {
+      return false;
+    }
+
+    internal::construct_at<Base>(this, expr.data(), rows, cols);
+    internal::construct_at(&m_stride, (StrideType::OuterStrideAtCompileTime == 0) ? 0 : outer_stride,
+                           (StrideType::InnerStrideAtCompileTime == 0) ? 0 : inner_stride);
+    return true;
+  }
+
+  StrideBase m_stride;
+};
+
+/** \class Ref
+ * \ingroup Core_Module
+ *
+ * \brief A matrix or vector expression mapping an existing expression
+ *
+ * \tparam PlainObjectType the equivalent matrix type of the mapped data
+ * \tparam Options specifies the pointer alignment in bytes. It can be: \c #Aligned128, , \c #Aligned64, \c #Aligned32,
+ * \c #Aligned16, \c #Aligned8 or \c #Unaligned. The default is \c #Unaligned. \tparam StrideType optionally specifies
+ * strides. By default, Ref implies a contiguous storage along the inner dimension (inner stride==1), but accepts a
+ * variable outer stride (leading dimension). This can be overridden by specifying strides. The type passed here must be
+ * a specialization of the Stride template, see examples below.
+ *
+ * This class provides a way to write non-template functions taking Eigen objects as parameters while limiting the
+ * number of copies. A Ref<> object can represent either a const expression or a l-value: \code
+ * // in-out argument:
+ * void foo1(Ref<VectorXf> x);
+ *
+ * // read-only const argument:
+ * void foo2(const Ref<const VectorXf>& x);
+ * \endcode
+ *
+ * In the in-out case, the input argument must satisfy the constraints of the actual Ref<> type, otherwise a compilation
+ * issue will be triggered. By default, a Ref<VectorXf> can reference any dense vector expression of float having a
+ * contiguous memory layout. Likewise, a Ref<MatrixXf> can reference any column-major dense matrix expression of float
+ * whose column's elements are contiguously stored with the possibility to have a constant space in-between each column,
+ * i.e. the inner stride must be equal to 1, but the outer stride (or leading dimension) can be greater than the number
+ * of rows.
+ *
+ * In the const case, if the input expression does not match the above requirement, then it is evaluated into a
+ * temporary before being passed to the function. Here are some examples: \code MatrixXf A; VectorXf a; foo1(a.head());
+ * // OK foo1(A.col());              // OK foo1(A.row());              // Compilation error because here innerstride!=1
+ * foo2(A.row());              // Compilation error because A.row() is a 1xN object while foo2 is expecting a Nx1 object
+ * foo2(A.row().transpose());  // The row is copied into a contiguous temporary
+ * foo2(2*a);                  // The expression is evaluated into a temporary
+ * foo2(A.col().segment(2,4)); // No temporary
+ * \endcode
+ *
+ * The range of inputs that can be referenced without temporary can be enlarged using the last two template parameters.
+ * Here is an example accepting an innerstride!=1:
+ * \code
+ * // in-out argument:
+ * void foo3(Ref<VectorXf,0,InnerStride<> > x);
+ * foo3(A.row());              // OK
+ * \endcode
+ * The downside here is that the function foo3 might be significantly slower than foo1 because it won't be able to
+ * exploit vectorization, and will involve more expensive address computations even if the input is contiguously stored
+ * in memory. To overcome this issue, one might propose to overload internally calling a template function, e.g.: \code
+ * // in the .h:
+ * void foo(const Ref<MatrixXf>& A);
+ * void foo(const Ref<MatrixXf,0,Stride<> >& A);
+ *
+ * // in the .cpp:
+ * template<typename TypeOfA> void foo_impl(const TypeOfA& A) {
+ *     ... // crazy code goes here
+ * }
+ * void foo(const Ref<MatrixXf>& A) { foo_impl(A); }
+ * void foo(const Ref<MatrixXf,0,Stride<> >& A) { foo_impl(A); }
+ * \endcode
+ *
+ * See also the following stackoverflow questions for further references:
+ *  - <a href="http://stackoverflow.com/questions/21132538/correct-usage-of-the-eigenref-class">Correct usage of the
+ * Eigen::Ref<> class</a>
+ *
+ * \sa PlainObjectBase::Map(), \ref TopicStorageOrders
+ */
+template <typename PlainObjectType, int Options, typename StrideType>
+class Ref : public RefBase<Ref<PlainObjectType, Options, StrideType> > {
+ private:
+  typedef internal::traits<Ref> Traits;
+  template <typename Derived>
+  EIGEN_DEVICE_FUNC inline Ref(
+      const PlainObjectBase<Derived>& expr,
+      std::enable_if_t<bool(Traits::template match<Derived>::MatchAtCompileTime), Derived>* = 0);
+
+ public:
+  typedef RefBase<Ref> Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Ref)
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  template <typename Derived>
+  EIGEN_DEVICE_FUNC inline Ref(
+      PlainObjectBase<Derived>& expr,
+      std::enable_if_t<bool(Traits::template match<Derived>::MatchAtCompileTime), Derived>* = 0) {
+    EIGEN_STATIC_ASSERT(bool(Traits::template match<Derived>::MatchAtCompileTime), STORAGE_LAYOUT_DOES_NOT_MATCH);
+    // Construction must pass since we will not create temporary storage in the non-const case.
+    const bool success = Base::construct(expr.derived());
+    EIGEN_UNUSED_VARIABLE(success)
+    eigen_assert(success);
+  }
+  template <typename Derived>
+  EIGEN_DEVICE_FUNC inline Ref(
+      const DenseBase<Derived>& expr,
+      std::enable_if_t<bool(Traits::template match<Derived>::MatchAtCompileTime), Derived>* = 0)
+#else
+  /** Implicit constructor from any dense expression */
+  template <typename Derived>
+  inline Ref(DenseBase<Derived>& expr)
+#endif
+  {
+    EIGEN_STATIC_ASSERT(bool(internal::is_lvalue<Derived>::value), THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY);
+    EIGEN_STATIC_ASSERT(bool(Traits::template match<Derived>::MatchAtCompileTime), STORAGE_LAYOUT_DOES_NOT_MATCH);
+    EIGEN_STATIC_ASSERT(!Derived::IsPlainObjectBase, THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY);
+    // Construction must pass since we will not create temporary storage in the non-const case.
+    const bool success = Base::construct(expr.const_cast_derived());
+    EIGEN_UNUSED_VARIABLE(success)
+    eigen_assert(success);
+  }
+
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Ref)
+};
+
+// this is the const ref version
+template <typename TPlainObjectType, int Options, typename StrideType>
+class Ref<const TPlainObjectType, Options, StrideType>
+    : public RefBase<Ref<const TPlainObjectType, Options, StrideType> > {
+  typedef internal::traits<Ref> Traits;
+
+  static constexpr bool may_map_m_object_successfully =
+      (static_cast<int>(StrideType::InnerStrideAtCompileTime) == 0 ||
+       static_cast<int>(StrideType::InnerStrideAtCompileTime) == 1 ||
+       static_cast<int>(StrideType::InnerStrideAtCompileTime) == Dynamic) &&
+      (TPlainObjectType::IsVectorAtCompileTime || static_cast<int>(StrideType::OuterStrideAtCompileTime) == 0 ||
+       static_cast<int>(StrideType::OuterStrideAtCompileTime) == Dynamic ||
+       static_cast<int>(StrideType::OuterStrideAtCompileTime) ==
+           static_cast<int>(TPlainObjectType::InnerSizeAtCompileTime) ||
+       static_cast<int>(TPlainObjectType::InnerSizeAtCompileTime) == Dynamic);
+
+ public:
+  typedef RefBase<Ref> Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Ref)
+
+  template <typename Derived>
+  EIGEN_DEVICE_FUNC inline Ref(const DenseBase<Derived>& expr,
+                               std::enable_if_t<bool(Traits::template match<Derived>::ScalarTypeMatch), Derived>* = 0) {
+    //      std::cout << match_helper<Derived>::HasDirectAccess << "," << match_helper<Derived>::OuterStrideMatch << ","
+    //      << match_helper<Derived>::InnerStrideMatch << "\n"; std::cout << int(StrideType::OuterStrideAtCompileTime)
+    //      << " - " << int(Derived::OuterStrideAtCompileTime) << "\n"; std::cout <<
+    //      int(StrideType::InnerStrideAtCompileTime) << " - " << int(Derived::InnerStrideAtCompileTime) << "\n";
+    EIGEN_STATIC_ASSERT(Traits::template match<Derived>::type::value || may_map_m_object_successfully,
+                        STORAGE_LAYOUT_DOES_NOT_MATCH);
+    construct(expr.derived(), typename Traits::template match<Derived>::type());
+  }
+
+  EIGEN_DEVICE_FUNC inline Ref(const Ref& other) : Base(other) {
+    // copy constructor shall not copy the m_object, to avoid unnecessary malloc and copy
+  }
+
+  EIGEN_DEVICE_FUNC inline Ref(Ref&& other) {
+    if (other.data() == other.m_object.data()) {
+      m_object = std::move(other.m_object);
+      Base::construct(m_object);
+    } else
+      Base::construct(other);
+  }
+
+  template <typename OtherRef>
+  EIGEN_DEVICE_FUNC inline Ref(const RefBase<OtherRef>& other) {
+    EIGEN_STATIC_ASSERT(Traits::template match<OtherRef>::type::value || may_map_m_object_successfully,
+                        STORAGE_LAYOUT_DOES_NOT_MATCH);
+    construct(other.derived(), typename Traits::template match<OtherRef>::type());
+  }
+
+ protected:
+  template <typename Expression>
+  EIGEN_DEVICE_FUNC void construct(const Expression& expr, internal::true_type) {
+    // Check if we can use the underlying expr's storage directly, otherwise call the copy version.
+    if (!Base::construct(expr)) {
+      construct(expr, internal::false_type());
+    }
+  }
+
+  template <typename Expression>
+  EIGEN_DEVICE_FUNC void construct(const Expression& expr, internal::false_type) {
+    internal::call_assignment_no_alias(m_object, expr, internal::assign_op<Scalar, Scalar>());
+    const bool success = Base::construct(m_object);
+    EIGEN_ONLY_USED_FOR_DEBUG(success)
+    eigen_assert(success);
+  }
+
+ protected:
+  TPlainObjectType m_object;
+};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_REF_H

include/Eigen/src/Core/Replicate.h

@@ -0,0 +1,130 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_REPLICATE_H
+#define EIGEN_REPLICATE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename MatrixType, int RowFactor, int ColFactor>
+struct traits<Replicate<MatrixType, RowFactor, ColFactor> > : traits<MatrixType> {
+  typedef typename MatrixType::Scalar Scalar;
+  typedef typename traits<MatrixType>::StorageKind StorageKind;
+  typedef typename traits<MatrixType>::XprKind XprKind;
+  typedef typename ref_selector<MatrixType>::type MatrixTypeNested;
+  typedef std::remove_reference_t<MatrixTypeNested> MatrixTypeNested_;
+  enum {
+    RowsAtCompileTime = RowFactor == Dynamic || int(MatrixType::RowsAtCompileTime) == Dynamic
+                            ? Dynamic
+                            : RowFactor * MatrixType::RowsAtCompileTime,
+    ColsAtCompileTime = ColFactor == Dynamic || int(MatrixType::ColsAtCompileTime) == Dynamic
+                            ? Dynamic
+                            : ColFactor * MatrixType::ColsAtCompileTime,
+    // FIXME we don't propagate the max sizes !!!
+    MaxRowsAtCompileTime = RowsAtCompileTime,
+    MaxColsAtCompileTime = ColsAtCompileTime,
+    IsRowMajor = MaxRowsAtCompileTime == 1 && MaxColsAtCompileTime != 1   ? 1
+                 : MaxColsAtCompileTime == 1 && MaxRowsAtCompileTime != 1 ? 0
+                 : (MatrixType::Flags & RowMajorBit)                      ? 1
+                                                                          : 0,
+
+    // FIXME enable DirectAccess with negative strides?
+    Flags = IsRowMajor ? RowMajorBit : 0
+  };
+};
+}  // namespace internal
+
+/**
+ * \class Replicate
+ * \ingroup Core_Module
+ *
+ * \brief Expression of the multiple replication of a matrix or vector
+ *
+ * \tparam MatrixType the type of the object we are replicating
+ * \tparam RowFactor number of repetitions at compile time along the vertical direction, can be Dynamic.
+ * \tparam ColFactor number of repetitions at compile time along the horizontal direction, can be Dynamic.
+ *
+ * This class represents an expression of the multiple replication of a matrix or vector.
+ * It is the return type of DenseBase::replicate() and most of the time
+ * this is the only way it is used.
+ *
+ * \sa DenseBase::replicate()
+ */
+template <typename MatrixType, int RowFactor, int ColFactor>
+class Replicate : public internal::dense_xpr_base<Replicate<MatrixType, RowFactor, ColFactor> >::type {
+  typedef typename internal::traits<Replicate>::MatrixTypeNested MatrixTypeNested;
+  typedef typename internal::traits<Replicate>::MatrixTypeNested_ MatrixTypeNested_;
+
+ public:
+  typedef typename internal::dense_xpr_base<Replicate>::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Replicate)
+  typedef internal::remove_all_t<MatrixType> NestedExpression;
+
+  template <typename OriginalMatrixType>
+  EIGEN_DEVICE_FUNC inline explicit Replicate(const OriginalMatrixType& matrix)
+      : m_matrix(matrix), m_rowFactor(RowFactor), m_colFactor(ColFactor) {
+    EIGEN_STATIC_ASSERT((internal::is_same<std::remove_const_t<MatrixType>, OriginalMatrixType>::value),
+                        THE_MATRIX_OR_EXPRESSION_THAT_YOU_PASSED_DOES_NOT_HAVE_THE_EXPECTED_TYPE)
+    eigen_assert(RowFactor != Dynamic && ColFactor != Dynamic);
+  }
+
+  template <typename OriginalMatrixType>
+  EIGEN_DEVICE_FUNC inline Replicate(const OriginalMatrixType& matrix, Index rowFactor, Index colFactor)
+      : m_matrix(matrix), m_rowFactor(rowFactor), m_colFactor(colFactor) {
+    EIGEN_STATIC_ASSERT((internal::is_same<std::remove_const_t<MatrixType>, OriginalMatrixType>::value),
+                        THE_MATRIX_OR_EXPRESSION_THAT_YOU_PASSED_DOES_NOT_HAVE_THE_EXPECTED_TYPE)
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const { return m_matrix.rows() * m_rowFactor.value(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const { return m_matrix.cols() * m_colFactor.value(); }
+
+  EIGEN_DEVICE_FUNC const MatrixTypeNested_& nestedExpression() const { return m_matrix; }
+
+ protected:
+  MatrixTypeNested m_matrix;
+  const internal::variable_if_dynamic<Index, RowFactor> m_rowFactor;
+  const internal::variable_if_dynamic<Index, ColFactor> m_colFactor;
+};
+
+/**
+ * \return an expression of the replication of \c *this
+ *
+ * Example: \include MatrixBase_replicate.cpp
+ * Output: \verbinclude MatrixBase_replicate.out
+ *
+ * \sa VectorwiseOp::replicate(), DenseBase::replicate(Index,Index), class Replicate
+ */
+template <typename Derived>
+template <int RowFactor, int ColFactor>
+EIGEN_DEVICE_FUNC const Replicate<Derived, RowFactor, ColFactor> DenseBase<Derived>::replicate() const {
+  return Replicate<Derived, RowFactor, ColFactor>(derived());
+}
+
+/**
+ * \return an expression of the replication of each column (or row) of \c *this
+ *
+ * Example: \include DirectionWise_replicate_int.cpp
+ * Output: \verbinclude DirectionWise_replicate_int.out
+ *
+ * \sa VectorwiseOp::replicate(), DenseBase::replicate(), class Replicate
+ */
+template <typename ExpressionType, int Direction>
+EIGEN_DEVICE_FUNC const typename VectorwiseOp<ExpressionType, Direction>::ReplicateReturnType
+VectorwiseOp<ExpressionType, Direction>::replicate(Index factor) const {
+  return typename VectorwiseOp<ExpressionType, Direction>::ReplicateReturnType(
+      _expression(), Direction == Vertical ? factor : 1, Direction == Horizontal ? factor : 1);
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_REPLICATE_H

include/Eigen/src/Core/Reshaped.h

@@ -0,0 +1,398 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2017 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2014 yoco <peter.xiau@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_RESHAPED_H
+#define EIGEN_RESHAPED_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class Reshaped
+ * \ingroup Core_Module
+ *
+ * \brief Expression of a fixed-size or dynamic-size reshape
+ *
+ * \tparam XprType the type of the expression in which we are taking a reshape
+ * \tparam Rows the number of rows of the reshape we are taking at compile time (optional)
+ * \tparam Cols the number of columns of the reshape we are taking at compile time (optional)
+ * \tparam Order can be ColMajor or RowMajor, default is ColMajor.
+ *
+ * This class represents an expression of either a fixed-size or dynamic-size reshape.
+ * It is the return type of DenseBase::reshaped(NRowsType,NColsType) and
+ * most of the time this is the only way it is used.
+ *
+ * If you want to directly manipulate reshaped expressions,
+ * for instance if you want to write a function returning such an expression,
+ * it is advised to use the \em auto keyword for such use cases.
+ *
+ * Here is an example illustrating the dynamic case:
+ * \include class_Reshaped.cpp
+ * Output: \verbinclude class_Reshaped.out
+ *
+ * Here is an example illustrating the fixed-size case:
+ * \include class_FixedReshaped.cpp
+ * Output: \verbinclude class_FixedReshaped.out
+ *
+ * \sa DenseBase::reshaped(NRowsType,NColsType)
+ */
+
+namespace internal {
+
+template <typename XprType, int Rows, int Cols, int Order>
+struct traits<Reshaped<XprType, Rows, Cols, Order> > : traits<XprType> {
+  typedef typename traits<XprType>::Scalar Scalar;
+  typedef typename traits<XprType>::StorageKind StorageKind;
+  typedef typename traits<XprType>::XprKind XprKind;
+  enum {
+    MatrixRows = traits<XprType>::RowsAtCompileTime,
+    MatrixCols = traits<XprType>::ColsAtCompileTime,
+    RowsAtCompileTime = Rows,
+    ColsAtCompileTime = Cols,
+    MaxRowsAtCompileTime = Rows,
+    MaxColsAtCompileTime = Cols,
+    XpxStorageOrder = ((int(traits<XprType>::Flags) & RowMajorBit) == RowMajorBit) ? RowMajor : ColMajor,
+    ReshapedStorageOrder = (RowsAtCompileTime == 1 && ColsAtCompileTime != 1)   ? RowMajor
+                           : (ColsAtCompileTime == 1 && RowsAtCompileTime != 1) ? ColMajor
+                                                                                : XpxStorageOrder,
+    HasSameStorageOrderAsXprType = (ReshapedStorageOrder == XpxStorageOrder),
+    InnerSize = (ReshapedStorageOrder == int(RowMajor)) ? int(ColsAtCompileTime) : int(RowsAtCompileTime),
+    InnerStrideAtCompileTime = HasSameStorageOrderAsXprType ? int(inner_stride_at_compile_time<XprType>::ret) : Dynamic,
+    OuterStrideAtCompileTime = Dynamic,
+
+    HasDirectAccess = internal::has_direct_access<XprType>::ret && (Order == int(XpxStorageOrder)) &&
+                      ((evaluator<XprType>::Flags & LinearAccessBit) == LinearAccessBit),
+
+    MaskPacketAccessBit =
+        (InnerSize == Dynamic || (InnerSize % packet_traits<Scalar>::size) == 0) && (InnerStrideAtCompileTime == 1)
+            ? PacketAccessBit
+            : 0,
+    // MaskAlignedBit = ((OuterStrideAtCompileTime!=Dynamic) && (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % 16)
+    // == 0)) ? AlignedBit : 0,
+    FlagsLinearAccessBit = (RowsAtCompileTime == 1 || ColsAtCompileTime == 1) ? LinearAccessBit : 0,
+    FlagsLvalueBit = is_lvalue<XprType>::value ? LvalueBit : 0,
+    FlagsRowMajorBit = (ReshapedStorageOrder == int(RowMajor)) ? RowMajorBit : 0,
+    FlagsDirectAccessBit = HasDirectAccess ? DirectAccessBit : 0,
+    Flags0 = traits<XprType>::Flags & ((HereditaryBits & ~RowMajorBit) | MaskPacketAccessBit),
+
+    Flags = (Flags0 | FlagsLinearAccessBit | FlagsLvalueBit | FlagsRowMajorBit | FlagsDirectAccessBit)
+  };
+};
+
+template <typename XprType, int Rows, int Cols, int Order, bool HasDirectAccess>
+class ReshapedImpl_dense;
+
+}  // end namespace internal
+
+template <typename XprType, int Rows, int Cols, int Order, typename StorageKind>
+class ReshapedImpl;
+
+template <typename XprType, int Rows, int Cols, int Order>
+class Reshaped : public ReshapedImpl<XprType, Rows, Cols, Order, typename internal::traits<XprType>::StorageKind> {
+  typedef ReshapedImpl<XprType, Rows, Cols, Order, typename internal::traits<XprType>::StorageKind> Impl;
+
+ public:
+  // typedef typename Impl::Base Base;
+  typedef Impl Base;
+  EIGEN_GENERIC_PUBLIC_INTERFACE(Reshaped)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Reshaped)
+
+  /** Fixed-size constructor
+   */
+  EIGEN_DEVICE_FUNC inline Reshaped(XprType& xpr) : Impl(xpr) {
+    EIGEN_STATIC_ASSERT(RowsAtCompileTime != Dynamic && ColsAtCompileTime != Dynamic,
+                        THIS_METHOD_IS_ONLY_FOR_FIXED_SIZE)
+    eigen_assert(Rows * Cols == xpr.rows() * xpr.cols());
+  }
+
+  /** Dynamic-size constructor
+   */
+  EIGEN_DEVICE_FUNC inline Reshaped(XprType& xpr, Index reshapeRows, Index reshapeCols)
+      : Impl(xpr, reshapeRows, reshapeCols) {
+    eigen_assert((RowsAtCompileTime == Dynamic || RowsAtCompileTime == reshapeRows) &&
+                 (ColsAtCompileTime == Dynamic || ColsAtCompileTime == reshapeCols));
+    eigen_assert(reshapeRows * reshapeCols == xpr.rows() * xpr.cols());
+  }
+};
+
+// The generic default implementation for dense reshape simply forward to the internal::ReshapedImpl_dense
+// that must be specialized for direct and non-direct access...
+template <typename XprType, int Rows, int Cols, int Order>
+class ReshapedImpl<XprType, Rows, Cols, Order, Dense>
+    : public internal::ReshapedImpl_dense<XprType, Rows, Cols, Order,
+                                          internal::traits<Reshaped<XprType, Rows, Cols, Order> >::HasDirectAccess> {
+  typedef internal::ReshapedImpl_dense<XprType, Rows, Cols, Order,
+                                       internal::traits<Reshaped<XprType, Rows, Cols, Order> >::HasDirectAccess>
+      Impl;
+
+ public:
+  typedef Impl Base;
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ReshapedImpl)
+  EIGEN_DEVICE_FUNC inline ReshapedImpl(XprType& xpr) : Impl(xpr) {}
+  EIGEN_DEVICE_FUNC inline ReshapedImpl(XprType& xpr, Index reshapeRows, Index reshapeCols)
+      : Impl(xpr, reshapeRows, reshapeCols) {}
+};
+
+namespace internal {
+
+/** \internal Internal implementation of dense Reshaped in the general case. */
+template <typename XprType, int Rows, int Cols, int Order>
+class ReshapedImpl_dense<XprType, Rows, Cols, Order, false>
+    : public internal::dense_xpr_base<Reshaped<XprType, Rows, Cols, Order> >::type {
+  typedef Reshaped<XprType, Rows, Cols, Order> ReshapedType;
+
+ public:
+  typedef typename internal::dense_xpr_base<ReshapedType>::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(ReshapedType)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ReshapedImpl_dense)
+
+  typedef typename internal::ref_selector<XprType>::non_const_type MatrixTypeNested;
+  typedef internal::remove_all_t<XprType> NestedExpression;
+
+  class InnerIterator;
+
+  /** Fixed-size constructor
+   */
+  EIGEN_DEVICE_FUNC inline ReshapedImpl_dense(XprType& xpr) : m_xpr(xpr), m_rows(Rows), m_cols(Cols) {}
+
+  /** Dynamic-size constructor
+   */
+  EIGEN_DEVICE_FUNC inline ReshapedImpl_dense(XprType& xpr, Index nRows, Index nCols)
+      : m_xpr(xpr), m_rows(nRows), m_cols(nCols) {}
+
+  EIGEN_DEVICE_FUNC Index rows() const { return m_rows; }
+  EIGEN_DEVICE_FUNC Index cols() const { return m_cols; }
+
+#ifdef EIGEN_PARSED_BY_DOXYGEN
+  /** \sa MapBase::data() */
+  EIGEN_DEVICE_FUNC constexpr const Scalar* data() const;
+  EIGEN_DEVICE_FUNC inline Index innerStride() const;
+  EIGEN_DEVICE_FUNC inline Index outerStride() const;
+#endif
+
+  /** \returns the nested expression */
+  EIGEN_DEVICE_FUNC const internal::remove_all_t<XprType>& nestedExpression() const { return m_xpr; }
+
+  /** \returns the nested expression */
+  EIGEN_DEVICE_FUNC std::remove_reference_t<XprType>& nestedExpression() { return m_xpr; }
+
+ protected:
+  MatrixTypeNested m_xpr;
+  const internal::variable_if_dynamic<Index, Rows> m_rows;
+  const internal::variable_if_dynamic<Index, Cols> m_cols;
+};
+
+/** \internal Internal implementation of dense Reshaped in the direct access case. */
+template <typename XprType, int Rows, int Cols, int Order>
+class ReshapedImpl_dense<XprType, Rows, Cols, Order, true> : public MapBase<Reshaped<XprType, Rows, Cols, Order> > {
+  typedef Reshaped<XprType, Rows, Cols, Order> ReshapedType;
+  typedef typename internal::ref_selector<XprType>::non_const_type XprTypeNested;
+
+ public:
+  typedef MapBase<ReshapedType> Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(ReshapedType)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ReshapedImpl_dense)
+
+  /** Fixed-size constructor
+   */
+  EIGEN_DEVICE_FUNC inline ReshapedImpl_dense(XprType& xpr) : Base(xpr.data()), m_xpr(xpr) {}
+
+  /** Dynamic-size constructor
+   */
+  EIGEN_DEVICE_FUNC inline ReshapedImpl_dense(XprType& xpr, Index nRows, Index nCols)
+      : Base(xpr.data(), nRows, nCols), m_xpr(xpr) {}
+
+  EIGEN_DEVICE_FUNC const internal::remove_all_t<XprTypeNested>& nestedExpression() const { return m_xpr; }
+
+  EIGEN_DEVICE_FUNC XprType& nestedExpression() { return m_xpr; }
+
+  /** \sa MapBase::innerStride() */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const { return m_xpr.innerStride(); }
+
+  /** \sa MapBase::outerStride() */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const {
+    return (((Flags & RowMajorBit) == RowMajorBit) ? this->cols() : this->rows()) * m_xpr.innerStride();
+  }
+
+ protected:
+  XprTypeNested m_xpr;
+};
+
+// Evaluators
+template <typename ArgType, int Rows, int Cols, int Order, bool HasDirectAccess>
+struct reshaped_evaluator;
+
+template <typename ArgType, int Rows, int Cols, int Order>
+struct evaluator<Reshaped<ArgType, Rows, Cols, Order> >
+    : reshaped_evaluator<ArgType, Rows, Cols, Order, traits<Reshaped<ArgType, Rows, Cols, Order> >::HasDirectAccess> {
+  typedef Reshaped<ArgType, Rows, Cols, Order> XprType;
+  typedef typename XprType::Scalar Scalar;
+  // TODO: should check for smaller packet types
+  typedef typename packet_traits<Scalar>::type PacketScalar;
+
+  enum {
+    CoeffReadCost = evaluator<ArgType>::CoeffReadCost,
+    HasDirectAccess = traits<XprType>::HasDirectAccess,
+
+    //     RowsAtCompileTime = traits<XprType>::RowsAtCompileTime,
+    //     ColsAtCompileTime = traits<XprType>::ColsAtCompileTime,
+    //     MaxRowsAtCompileTime = traits<XprType>::MaxRowsAtCompileTime,
+    //     MaxColsAtCompileTime = traits<XprType>::MaxColsAtCompileTime,
+    //
+    //     InnerStrideAtCompileTime = traits<XprType>::HasSameStorageOrderAsXprType
+    //                              ? int(inner_stride_at_compile_time<ArgType>::ret)
+    //                              : Dynamic,
+    //     OuterStrideAtCompileTime = Dynamic,
+
+    FlagsLinearAccessBit =
+        (traits<XprType>::RowsAtCompileTime == 1 || traits<XprType>::ColsAtCompileTime == 1 || HasDirectAccess)
+            ? LinearAccessBit
+            : 0,
+    FlagsRowMajorBit = (traits<XprType>::ReshapedStorageOrder == int(RowMajor)) ? RowMajorBit : 0,
+    FlagsDirectAccessBit = HasDirectAccess ? DirectAccessBit : 0,
+    Flags0 = evaluator<ArgType>::Flags & (HereditaryBits & ~RowMajorBit),
+    Flags = Flags0 | FlagsLinearAccessBit | FlagsRowMajorBit | FlagsDirectAccessBit,
+
+    PacketAlignment = unpacket_traits<PacketScalar>::alignment,
+    Alignment = evaluator<ArgType>::Alignment
+  };
+  typedef reshaped_evaluator<ArgType, Rows, Cols, Order, HasDirectAccess> reshaped_evaluator_type;
+  EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : reshaped_evaluator_type(xpr) {
+    EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
+  }
+};
+
+template <typename ArgType, int Rows, int Cols, int Order>
+struct reshaped_evaluator<ArgType, Rows, Cols, Order, /* HasDirectAccess */ false>
+    : evaluator_base<Reshaped<ArgType, Rows, Cols, Order> > {
+  typedef Reshaped<ArgType, Rows, Cols, Order> XprType;
+
+  enum {
+    CoeffReadCost = evaluator<ArgType>::CoeffReadCost /* TODO + cost of index computations */,
+
+    Flags = (evaluator<ArgType>::Flags & (HereditaryBits /*| LinearAccessBit | DirectAccessBit*/)),
+
+    Alignment = 0
+  };
+
+  EIGEN_DEVICE_FUNC explicit reshaped_evaluator(const XprType& xpr) : m_argImpl(xpr.nestedExpression()), m_xpr(xpr) {
+    EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
+  }
+
+  typedef typename XprType::Scalar Scalar;
+  typedef typename XprType::CoeffReturnType CoeffReturnType;
+
+  typedef std::pair<Index, Index> RowCol;
+
+  EIGEN_DEVICE_FUNC inline RowCol index_remap(Index rowId, Index colId) const {
+    if (Order == ColMajor) {
+      const Index nth_elem_idx = colId * m_xpr.rows() + rowId;
+      return RowCol(nth_elem_idx % m_xpr.nestedExpression().rows(), nth_elem_idx / m_xpr.nestedExpression().rows());
+    } else {
+      const Index nth_elem_idx = colId + rowId * m_xpr.cols();
+      return RowCol(nth_elem_idx / m_xpr.nestedExpression().cols(), nth_elem_idx % m_xpr.nestedExpression().cols());
+    }
+  }
+
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index rowId, Index colId) {
+    EIGEN_STATIC_ASSERT_LVALUE(XprType)
+    const RowCol row_col = index_remap(rowId, colId);
+    return m_argImpl.coeffRef(row_col.first, row_col.second);
+  }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index rowId, Index colId) const {
+    const RowCol row_col = index_remap(rowId, colId);
+    return m_argImpl.coeffRef(row_col.first, row_col.second);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const CoeffReturnType coeff(Index rowId, Index colId) const {
+    const RowCol row_col = index_remap(rowId, colId);
+    return m_argImpl.coeff(row_col.first, row_col.second);
+  }
+
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index index) {
+    EIGEN_STATIC_ASSERT_LVALUE(XprType)
+    const RowCol row_col = index_remap(Rows == 1 ? 0 : index, Rows == 1 ? index : 0);
+    return m_argImpl.coeffRef(row_col.first, row_col.second);
+  }
+
+  EIGEN_DEVICE_FUNC inline const Scalar& coeffRef(Index index) const {
+    const RowCol row_col = index_remap(Rows == 1 ? 0 : index, Rows == 1 ? index : 0);
+    return m_argImpl.coeffRef(row_col.first, row_col.second);
+  }
+
+  EIGEN_DEVICE_FUNC inline const CoeffReturnType coeff(Index index) const {
+    const RowCol row_col = index_remap(Rows == 1 ? 0 : index, Rows == 1 ? index : 0);
+    return m_argImpl.coeff(row_col.first, row_col.second);
+  }
+#if 0
+  EIGEN_DEVICE_FUNC
+  template<int LoadMode>
+  inline PacketScalar packet(Index rowId, Index colId) const
+  {
+    const RowCol row_col = index_remap(rowId, colId);
+    return m_argImpl.template packet<Unaligned>(row_col.first, row_col.second);
+
+  }
+
+  template<int LoadMode>
+  EIGEN_DEVICE_FUNC
+  inline void writePacket(Index rowId, Index colId, const PacketScalar& val)
+  {
+    const RowCol row_col = index_remap(rowId, colId);
+    m_argImpl.const_cast_derived().template writePacket<Unaligned>
+            (row_col.first, row_col.second, val);
+  }
+
+  template<int LoadMode>
+  EIGEN_DEVICE_FUNC
+  inline PacketScalar packet(Index index) const
+  {
+    const RowCol row_col = index_remap(RowsAtCompileTime == 1 ? 0 : index,
+                                        RowsAtCompileTime == 1 ? index : 0);
+    return m_argImpl.template packet<Unaligned>(row_col.first, row_col.second);
+  }
+
+  template<int LoadMode>
+  EIGEN_DEVICE_FUNC
+  inline void writePacket(Index index, const PacketScalar& val)
+  {
+    const RowCol row_col = index_remap(RowsAtCompileTime == 1 ? 0 : index,
+                                        RowsAtCompileTime == 1 ? index : 0);
+    return m_argImpl.template packet<Unaligned>(row_col.first, row_col.second, val);
+  }
+#endif
+ protected:
+  evaluator<ArgType> m_argImpl;
+  const XprType& m_xpr;
+};
+
+template <typename ArgType, int Rows, int Cols, int Order>
+struct reshaped_evaluator<ArgType, Rows, Cols, Order, /* HasDirectAccess */ true>
+    : mapbase_evaluator<Reshaped<ArgType, Rows, Cols, Order>,
+                        typename Reshaped<ArgType, Rows, Cols, Order>::PlainObject> {
+  typedef Reshaped<ArgType, Rows, Cols, Order> XprType;
+  typedef typename XprType::Scalar Scalar;
+
+  EIGEN_DEVICE_FUNC explicit reshaped_evaluator(const XprType& xpr)
+      : mapbase_evaluator<XprType, typename XprType::PlainObject>(xpr) {
+    // TODO: for the 3.4 release, this should be turned to an internal assertion, but let's keep it as is for the beta
+    // lifetime
+    eigen_assert(((std::uintptr_t(xpr.data()) % plain_enum_max(1, evaluator<XprType>::Alignment)) == 0) &&
+                 "data is not aligned");
+  }
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_RESHAPED_H

include/Eigen/src/Core/ReturnByValue.h

@@ -0,0 +1,115 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2009-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_RETURNBYVALUE_H
+#define EIGEN_RETURNBYVALUE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Derived>
+struct traits<ReturnByValue<Derived> > : public traits<typename traits<Derived>::ReturnType> {
+  enum {
+    // We're disabling the DirectAccess because e.g. the constructor of
+    // the Block-with-DirectAccess expression requires to have a coeffRef method.
+    // Also, we don't want to have to implement the stride stuff.
+    Flags = (traits<typename traits<Derived>::ReturnType>::Flags | EvalBeforeNestingBit) & ~DirectAccessBit
+  };
+};
+
+/* The ReturnByValue object doesn't even have a coeff() method.
+ * So the only way that nesting it in an expression can work, is by evaluating it into a plain matrix.
+ * So internal::nested always gives the plain return matrix type.
+ *
+ * FIXME: I don't understand why we need this specialization: isn't this taken care of by the EvalBeforeNestingBit ??
+ * Answer: EvalBeforeNestingBit should be deprecated since we have the evaluators
+ */
+template <typename Derived, int n, typename PlainObject>
+struct nested_eval<ReturnByValue<Derived>, n, PlainObject> {
+  typedef typename traits<Derived>::ReturnType type;
+};
+
+}  // end namespace internal
+
+/** \class ReturnByValue
+ * \ingroup Core_Module
+ *
+ */
+template <typename Derived>
+class ReturnByValue : public internal::dense_xpr_base<ReturnByValue<Derived> >::type, internal::no_assignment_operator {
+ public:
+  typedef typename internal::traits<Derived>::ReturnType ReturnType;
+
+  typedef typename internal::dense_xpr_base<ReturnByValue>::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(ReturnByValue)
+
+  template <typename Dest>
+  EIGEN_DEVICE_FUNC inline void evalTo(Dest& dst) const {
+    static_cast<const Derived*>(this)->evalTo(dst);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT {
+    return static_cast<const Derived*>(this)->rows();
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT {
+    return static_cast<const Derived*>(this)->cols();
+  }
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+#define Unusable \
+  YOU_ARE_TRYING_TO_ACCESS_A_SINGLE_COEFFICIENT_IN_A_SPECIAL_EXPRESSION_WHERE_THAT_IS_NOT_ALLOWED_BECAUSE_THAT_WOULD_BE_INEFFICIENT
+  class Unusable {
+    Unusable(const Unusable&) {}
+    Unusable& operator=(const Unusable&) { return *this; }
+  };
+  const Unusable& coeff(Index) const { return *reinterpret_cast<const Unusable*>(this); }
+  const Unusable& coeff(Index, Index) const { return *reinterpret_cast<const Unusable*>(this); }
+  Unusable& coeffRef(Index) { return *reinterpret_cast<Unusable*>(this); }
+  Unusable& coeffRef(Index, Index) { return *reinterpret_cast<Unusable*>(this); }
+#undef Unusable
+#endif
+};
+
+template <typename Derived>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC Derived& DenseBase<Derived>::operator=(const ReturnByValue<OtherDerived>& other) {
+  other.evalTo(derived());
+  return derived();
+}
+
+namespace internal {
+
+// Expression is evaluated in a temporary; default implementation of Assignment is bypassed so that
+// when a ReturnByValue expression is assigned, the evaluator is not constructed.
+// TODO: Finalize port to new regime; ReturnByValue should not exist in the expression world
+
+template <typename Derived>
+struct evaluator<ReturnByValue<Derived> > : public evaluator<typename internal::traits<Derived>::ReturnType> {
+  typedef ReturnByValue<Derived> XprType;
+  typedef typename internal::traits<Derived>::ReturnType PlainObject;
+  typedef evaluator<PlainObject> Base;
+
+  EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : m_result(xpr.rows(), xpr.cols()) {
+    internal::construct_at<Base>(this, m_result);
+    xpr.evalTo(m_result);
+  }
+
+ protected:
+  PlainObject m_result;
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_RETURNBYVALUE_H

include/Eigen/src/Core/Reverse.h

@@ -0,0 +1,202 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2009 Ricard Marxer <email@ricardmarxer.com>
+// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_REVERSE_H
+#define EIGEN_REVERSE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename MatrixType, int Direction>
+struct traits<Reverse<MatrixType, Direction> > : traits<MatrixType> {
+  typedef typename MatrixType::Scalar Scalar;
+  typedef typename traits<MatrixType>::StorageKind StorageKind;
+  typedef typename traits<MatrixType>::XprKind XprKind;
+  typedef typename ref_selector<MatrixType>::type MatrixTypeNested;
+  typedef std::remove_reference_t<MatrixTypeNested> MatrixTypeNested_;
+  enum {
+    RowsAtCompileTime = MatrixType::RowsAtCompileTime,
+    ColsAtCompileTime = MatrixType::ColsAtCompileTime,
+    MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime,
+    Flags = MatrixTypeNested_::Flags & (RowMajorBit | LvalueBit)
+  };
+};
+
+template <typename PacketType, bool ReversePacket>
+struct reverse_packet_cond {
+  static inline PacketType run(const PacketType& x) { return preverse(x); }
+};
+
+template <typename PacketType>
+struct reverse_packet_cond<PacketType, false> {
+  static inline PacketType run(const PacketType& x) { return x; }
+};
+
+}  // end namespace internal
+
+/** \class Reverse
+ * \ingroup Core_Module
+ *
+ * \brief Expression of the reverse of a vector or matrix
+ *
+ * \tparam MatrixType the type of the object of which we are taking the reverse
+ * \tparam Direction defines the direction of the reverse operation, can be Vertical, Horizontal, or BothDirections
+ *
+ * This class represents an expression of the reverse of a vector.
+ * It is the return type of MatrixBase::reverse() and VectorwiseOp::reverse()
+ * and most of the time this is the only way it is used.
+ *
+ * \sa MatrixBase::reverse(), VectorwiseOp::reverse()
+ */
+template <typename MatrixType, int Direction>
+class Reverse : public internal::dense_xpr_base<Reverse<MatrixType, Direction> >::type {
+ public:
+  typedef typename internal::dense_xpr_base<Reverse>::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Reverse)
+  typedef internal::remove_all_t<MatrixType> NestedExpression;
+  using Base::IsRowMajor;
+
+ protected:
+  enum {
+    PacketSize = internal::packet_traits<Scalar>::size,
+    IsColMajor = !IsRowMajor,
+    ReverseRow = (Direction == Vertical) || (Direction == BothDirections),
+    ReverseCol = (Direction == Horizontal) || (Direction == BothDirections),
+    OffsetRow = ReverseRow && IsColMajor ? PacketSize : 1,
+    OffsetCol = ReverseCol && IsRowMajor ? PacketSize : 1,
+    ReversePacket = (Direction == BothDirections) || ((Direction == Vertical) && IsColMajor) ||
+                    ((Direction == Horizontal) && IsRowMajor)
+  };
+  typedef internal::reverse_packet_cond<PacketScalar, ReversePacket> reverse_packet;
+
+ public:
+  EIGEN_DEVICE_FUNC explicit inline Reverse(const MatrixType& matrix) : m_matrix(matrix) {}
+
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Reverse)
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_matrix.cols(); }
+
+  EIGEN_DEVICE_FUNC inline Index innerStride() const { return -m_matrix.innerStride(); }
+
+  EIGEN_DEVICE_FUNC const internal::remove_all_t<typename MatrixType::Nested>& nestedExpression() const {
+    return m_matrix;
+  }
+
+ protected:
+  typename MatrixType::Nested m_matrix;
+};
+
+/** \returns an expression of the reverse of *this.
+ *
+ * Example: \include MatrixBase_reverse.cpp
+ * Output: \verbinclude MatrixBase_reverse.out
+ *
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline typename DenseBase<Derived>::ReverseReturnType DenseBase<Derived>::reverse() {
+  return ReverseReturnType(derived());
+}
+
+// reverse const overload moved DenseBase.h due to a CUDA compiler bug
+
+/** This is the "in place" version of reverse: it reverses \c *this.
+ *
+ * In most cases it is probably better to simply use the reversed expression
+ * of a matrix. However, when reversing the matrix data itself is really needed,
+ * then this "in-place" version is probably the right choice because it provides
+ * the following additional benefits:
+ *  - less error prone: doing the same operation with .reverse() requires special care:
+ *    \code m = m.reverse().eval(); \endcode
+ *  - this API enables reverse operations without the need for a temporary
+ *  - it allows future optimizations (cache friendliness, etc.)
+ *
+ * \sa VectorwiseOp::reverseInPlace(), reverse() */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline void DenseBase<Derived>::reverseInPlace() {
+  constexpr int HalfRowsAtCompileTime = RowsAtCompileTime == Dynamic ? Dynamic : RowsAtCompileTime / 2;
+  constexpr int HalfColsAtCompileTime = ColsAtCompileTime == Dynamic ? Dynamic : ColsAtCompileTime / 2;
+  if (cols() > rows()) {
+    Index half = cols() / 2;
+    this->template leftCols<HalfColsAtCompileTime>(half).swap(
+        this->template rightCols<HalfColsAtCompileTime>(half).reverse());
+    if ((cols() % 2) == 1) {
+      Index half2 = rows() / 2;
+      col(half).template head<HalfRowsAtCompileTime>(half2).swap(
+          col(half).template tail<HalfRowsAtCompileTime>(half2).reverse());
+    }
+  } else {
+    Index half = rows() / 2;
+    this->template topRows<HalfRowsAtCompileTime>(half).swap(
+        this->template bottomRows<HalfRowsAtCompileTime>(half).reverse());
+    if ((rows() % 2) == 1) {
+      Index half2 = cols() / 2;
+      row(half).template head<HalfColsAtCompileTime>(half2).swap(
+          row(half).template tail<HalfColsAtCompileTime>(half2).reverse());
+    }
+  }
+}
+
+namespace internal {
+
+template <int Direction>
+struct vectorwise_reverse_inplace_impl;
+
+template <>
+struct vectorwise_reverse_inplace_impl<Vertical> {
+  template <typename ExpressionType>
+  static void run(ExpressionType& xpr) {
+    constexpr Index HalfAtCompileTime =
+        ExpressionType::RowsAtCompileTime == Dynamic ? Dynamic : ExpressionType::RowsAtCompileTime / 2;
+    Index half = xpr.rows() / 2;
+    xpr.template topRows<HalfAtCompileTime>(half).swap(
+        xpr.template bottomRows<HalfAtCompileTime>(half).colwise().reverse());
+  }
+};
+
+template <>
+struct vectorwise_reverse_inplace_impl<Horizontal> {
+  template <typename ExpressionType>
+  static void run(ExpressionType& xpr) {
+    constexpr Index HalfAtCompileTime =
+        ExpressionType::ColsAtCompileTime == Dynamic ? Dynamic : ExpressionType::ColsAtCompileTime / 2;
+    Index half = xpr.cols() / 2;
+    xpr.template leftCols<HalfAtCompileTime>(half).swap(
+        xpr.template rightCols<HalfAtCompileTime>(half).rowwise().reverse());
+  }
+};
+
+}  // end namespace internal
+
+/** This is the "in place" version of VectorwiseOp::reverse: it reverses each column or row of \c *this.
+ *
+ * In most cases it is probably better to simply use the reversed expression
+ * of a matrix. However, when reversing the matrix data itself is really needed,
+ * then this "in-place" version is probably the right choice because it provides
+ * the following additional benefits:
+ *  - less error prone: doing the same operation with .reverse() requires special care:
+ *    \code m = m.reverse().eval(); \endcode
+ *  - this API enables reverse operations without the need for a temporary
+ *
+ * \sa DenseBase::reverseInPlace(), reverse() */
+template <typename ExpressionType, int Direction>
+EIGEN_DEVICE_FUNC void VectorwiseOp<ExpressionType, Direction>::reverseInPlace() {
+  internal::vectorwise_reverse_inplace_impl<Direction>::run(m_matrix);
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_REVERSE_H

include/Eigen/src/Core/Select.h

@@ -0,0 +1,156 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_SELECT_H
+#define EIGEN_SELECT_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class Select
+ * \ingroup Core_Module
+ *
+ * \brief Expression of a coefficient wise version of the C++ ternary operator ?:
+ *
+ * \tparam ConditionMatrixType the type of the \em condition expression which must be a boolean matrix
+ * \tparam ThenMatrixType the type of the \em then expression
+ * \tparam ElseMatrixType the type of the \em else expression
+ *
+ * This class represents an expression of a coefficient wise version of the C++ ternary operator ?:.
+ * It is the return type of DenseBase::select() and most of the time this is the only way it is used.
+ *
+ * \sa DenseBase::select(const DenseBase<ThenDerived>&, const DenseBase<ElseDerived>&) const
+ */
+
+namespace internal {
+template <typename ConditionMatrixType, typename ThenMatrixType, typename ElseMatrixType>
+struct traits<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> > : traits<ThenMatrixType> {
+  typedef typename traits<ThenMatrixType>::Scalar Scalar;
+  typedef Dense StorageKind;
+  typedef typename traits<ThenMatrixType>::XprKind XprKind;
+  typedef typename ConditionMatrixType::Nested ConditionMatrixNested;
+  typedef typename ThenMatrixType::Nested ThenMatrixNested;
+  typedef typename ElseMatrixType::Nested ElseMatrixNested;
+  enum {
+    RowsAtCompileTime = ConditionMatrixType::RowsAtCompileTime,
+    ColsAtCompileTime = ConditionMatrixType::ColsAtCompileTime,
+    MaxRowsAtCompileTime = ConditionMatrixType::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = ConditionMatrixType::MaxColsAtCompileTime,
+    Flags = (unsigned int)ThenMatrixType::Flags & ElseMatrixType::Flags & RowMajorBit
+  };
+};
+}  // namespace internal
+
+template <typename ConditionMatrixType, typename ThenMatrixType, typename ElseMatrixType>
+class Select : public internal::dense_xpr_base<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> >::type,
+               internal::no_assignment_operator {
+ public:
+  typedef typename internal::dense_xpr_base<Select>::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Select)
+
+  inline EIGEN_DEVICE_FUNC Select(const ConditionMatrixType& a_conditionMatrix, const ThenMatrixType& a_thenMatrix,
+                                  const ElseMatrixType& a_elseMatrix)
+      : m_condition(a_conditionMatrix), m_then(a_thenMatrix), m_else(a_elseMatrix) {
+    eigen_assert(m_condition.rows() == m_then.rows() && m_condition.rows() == m_else.rows());
+    eigen_assert(m_condition.cols() == m_then.cols() && m_condition.cols() == m_else.cols());
+  }
+
+  inline EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_condition.rows(); }
+  inline EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_condition.cols(); }
+
+  inline EIGEN_DEVICE_FUNC const Scalar coeff(Index i, Index j) const {
+    if (m_condition.coeff(i, j))
+      return m_then.coeff(i, j);
+    else
+      return m_else.coeff(i, j);
+  }
+
+  inline EIGEN_DEVICE_FUNC const Scalar coeff(Index i) const {
+    if (m_condition.coeff(i))
+      return m_then.coeff(i);
+    else
+      return m_else.coeff(i);
+  }
+
+  inline EIGEN_DEVICE_FUNC const ConditionMatrixType& conditionMatrix() const { return m_condition; }
+
+  inline EIGEN_DEVICE_FUNC const ThenMatrixType& thenMatrix() const { return m_then; }
+
+  inline EIGEN_DEVICE_FUNC const ElseMatrixType& elseMatrix() const { return m_else; }
+
+ protected:
+  typename ConditionMatrixType::Nested m_condition;
+  typename ThenMatrixType::Nested m_then;
+  typename ElseMatrixType::Nested m_else;
+};
+
+/** \returns a matrix where each coefficient (i,j) is equal to \a thenMatrix(i,j)
+ * if \c *this(i,j) != Scalar(0), and \a elseMatrix(i,j) otherwise.
+ *
+ * Example: \include MatrixBase_select.cpp
+ * Output: \verbinclude MatrixBase_select.out
+ *
+ * \sa DenseBase::bitwiseSelect(const DenseBase<ThenDerived>&, const DenseBase<ElseDerived>&)
+ */
+template <typename Derived>
+template <typename ThenDerived, typename ElseDerived>
+inline EIGEN_DEVICE_FUNC CwiseTernaryOp<
+    internal::scalar_boolean_select_op<typename DenseBase<ThenDerived>::Scalar, typename DenseBase<ElseDerived>::Scalar,
+                                       typename DenseBase<Derived>::Scalar>,
+    ThenDerived, ElseDerived, Derived>
+DenseBase<Derived>::select(const DenseBase<ThenDerived>& thenMatrix, const DenseBase<ElseDerived>& elseMatrix) const {
+  using Op = internal::scalar_boolean_select_op<typename DenseBase<ThenDerived>::Scalar,
+                                                typename DenseBase<ElseDerived>::Scalar, Scalar>;
+  return CwiseTernaryOp<Op, ThenDerived, ElseDerived, Derived>(thenMatrix.derived(), elseMatrix.derived(), derived(),
+                                                               Op());
+}
+/** Version of DenseBase::select(const DenseBase&, const DenseBase&) with
+ * the \em else expression being a scalar value.
+ *
+ * \sa DenseBase::booleanSelect(const DenseBase<ThenDerived>&, const DenseBase<ElseDerived>&) const, class Select
+ */
+template <typename Derived>
+template <typename ThenDerived>
+inline EIGEN_DEVICE_FUNC CwiseTernaryOp<
+    internal::scalar_boolean_select_op<typename DenseBase<ThenDerived>::Scalar, typename DenseBase<ThenDerived>::Scalar,
+                                       typename DenseBase<Derived>::Scalar>,
+    ThenDerived, typename DenseBase<ThenDerived>::ConstantReturnType, Derived>
+DenseBase<Derived>::select(const DenseBase<ThenDerived>& thenMatrix,
+                           const typename DenseBase<ThenDerived>::Scalar& elseScalar) const {
+  using ElseConstantType = typename DenseBase<ThenDerived>::ConstantReturnType;
+  using Op = internal::scalar_boolean_select_op<typename DenseBase<ThenDerived>::Scalar,
+                                                typename DenseBase<ThenDerived>::Scalar, Scalar>;
+  return CwiseTernaryOp<Op, ThenDerived, ElseConstantType, Derived>(
+      thenMatrix.derived(), ElseConstantType(rows(), cols(), elseScalar), derived(), Op());
+}
+/** Version of DenseBase::select(const DenseBase&, const DenseBase&) with
+ * the \em then expression being a scalar value.
+ *
+ * \sa DenseBase::booleanSelect(const DenseBase<ThenDerived>&, const DenseBase<ElseDerived>&) const, class Select
+ */
+template <typename Derived>
+template <typename ElseDerived>
+inline EIGEN_DEVICE_FUNC CwiseTernaryOp<
+    internal::scalar_boolean_select_op<typename DenseBase<ElseDerived>::Scalar, typename DenseBase<ElseDerived>::Scalar,
+                                       typename DenseBase<Derived>::Scalar>,
+    typename DenseBase<ElseDerived>::ConstantReturnType, ElseDerived, Derived>
+DenseBase<Derived>::select(const typename DenseBase<ElseDerived>::Scalar& thenScalar,
+                           const DenseBase<ElseDerived>& elseMatrix) const {
+  using ThenConstantType = typename DenseBase<ElseDerived>::ConstantReturnType;
+  using Op = internal::scalar_boolean_select_op<typename DenseBase<ElseDerived>::Scalar,
+                                                typename DenseBase<ElseDerived>::Scalar, Scalar>;
+  return CwiseTernaryOp<Op, ThenConstantType, ElseDerived, Derived>(ThenConstantType(rows(), cols(), thenScalar),
+                                                                    elseMatrix.derived(), derived(), Op());
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_SELECT_H

include/Eigen/src/Core/SelfAdjointView.h

@@ -0,0 +1,329 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_SELFADJOINTMATRIX_H
+#define EIGEN_SELFADJOINTMATRIX_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class SelfAdjointView
+ * \ingroup Core_Module
+ *
+ *
+ * \brief Expression of a selfadjoint matrix from a triangular part of a dense matrix
+ *
+ * \tparam MatrixType the type of the dense matrix storing the coefficients
+ * \tparam TriangularPart can be either \c #Lower or \c #Upper
+ *
+ * This class is an expression of a sefladjoint matrix from a triangular part of a matrix
+ * with given dense storage of the coefficients. It is the return type of MatrixBase::selfadjointView()
+ * and most of the time this is the only way that it is used.
+ *
+ * \sa class TriangularBase, MatrixBase::selfadjointView()
+ */
+
+namespace internal {
+template <typename MatrixType, unsigned int UpLo>
+struct traits<SelfAdjointView<MatrixType, UpLo> > : traits<MatrixType> {
+  typedef typename ref_selector<MatrixType>::non_const_type MatrixTypeNested;
+  typedef remove_all_t<MatrixTypeNested> MatrixTypeNestedCleaned;
+  typedef MatrixType ExpressionType;
+  typedef typename MatrixType::PlainObject FullMatrixType;
+  enum {
+    Mode = UpLo | SelfAdjoint,
+    FlagsLvalueBit = is_lvalue<MatrixType>::value ? LvalueBit : 0,
+    Flags = MatrixTypeNestedCleaned::Flags & (HereditaryBits | FlagsLvalueBit) &
+            (~(PacketAccessBit | DirectAccessBit | LinearAccessBit))  // FIXME these flags should be preserved
+  };
+};
+}  // namespace internal
+
+template <typename MatrixType_, unsigned int UpLo>
+class SelfAdjointView : public TriangularBase<SelfAdjointView<MatrixType_, UpLo> > {
+ public:
+  EIGEN_STATIC_ASSERT(UpLo == Lower || UpLo == Upper, SELFADJOINTVIEW_ACCEPTS_UPPER_AND_LOWER_MODE_ONLY)
+
+  typedef MatrixType_ MatrixType;
+  typedef TriangularBase<SelfAdjointView> Base;
+  typedef typename internal::traits<SelfAdjointView>::MatrixTypeNested MatrixTypeNested;
+  typedef typename internal::traits<SelfAdjointView>::MatrixTypeNestedCleaned MatrixTypeNestedCleaned;
+  typedef MatrixTypeNestedCleaned NestedExpression;
+
+  /** \brief The type of coefficients in this matrix */
+  typedef typename internal::traits<SelfAdjointView>::Scalar Scalar;
+  typedef typename MatrixType::StorageIndex StorageIndex;
+  typedef internal::remove_all_t<typename MatrixType::ConjugateReturnType> MatrixConjugateReturnType;
+  typedef SelfAdjointView<std::add_const_t<MatrixType>, UpLo> ConstSelfAdjointView;
+
+  enum {
+    Mode = internal::traits<SelfAdjointView>::Mode,
+    Flags = internal::traits<SelfAdjointView>::Flags,
+    TransposeMode = ((int(Mode) & int(Upper)) ? Lower : 0) | ((int(Mode) & int(Lower)) ? Upper : 0)
+  };
+  typedef typename MatrixType::PlainObject PlainObject;
+
+  EIGEN_DEVICE_FUNC explicit inline SelfAdjointView(MatrixType& matrix) : m_matrix(matrix) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_matrix.cols(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return m_matrix.outerStride(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return m_matrix.innerStride(); }
+
+  /** \sa MatrixBase::coeff()
+   * \warning the coordinates must fit into the referenced triangular part
+   */
+  EIGEN_DEVICE_FUNC inline Scalar coeff(Index row, Index col) const {
+    Base::check_coordinates_internal(row, col);
+    return m_matrix.coeff(row, col);
+  }
+
+  /** \sa MatrixBase::coeffRef()
+   * \warning the coordinates must fit into the referenced triangular part
+   */
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index row, Index col) {
+    EIGEN_STATIC_ASSERT_LVALUE(SelfAdjointView);
+    Base::check_coordinates_internal(row, col);
+    return m_matrix.coeffRef(row, col);
+  }
+
+  /** \internal */
+  EIGEN_DEVICE_FUNC const MatrixTypeNestedCleaned& _expression() const { return m_matrix; }
+
+  EIGEN_DEVICE_FUNC const MatrixTypeNestedCleaned& nestedExpression() const { return m_matrix; }
+  EIGEN_DEVICE_FUNC MatrixTypeNestedCleaned& nestedExpression() { return m_matrix; }
+
+  /** Efficient triangular matrix times vector/matrix product */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC const Product<SelfAdjointView, OtherDerived> operator*(const MatrixBase<OtherDerived>& rhs) const {
+    return Product<SelfAdjointView, OtherDerived>(*this, rhs.derived());
+  }
+
+  /** Efficient vector/matrix times triangular matrix product */
+  template <typename OtherDerived>
+  friend EIGEN_DEVICE_FUNC const Product<OtherDerived, SelfAdjointView> operator*(const MatrixBase<OtherDerived>& lhs,
+                                                                                  const SelfAdjointView& rhs) {
+    return Product<OtherDerived, SelfAdjointView>(lhs.derived(), rhs);
+  }
+
+  friend EIGEN_DEVICE_FUNC const
+      SelfAdjointView<const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(Scalar, MatrixType, product), UpLo>
+      operator*(const Scalar& s, const SelfAdjointView& mat) {
+    return (s * mat.nestedExpression()).template selfadjointView<UpLo>();
+  }
+
+  /** Perform a symmetric rank 2 update of the selfadjoint matrix \c *this:
+   * \f$ this = this + \alpha u v^* + conj(\alpha) v u^* \f$
+   * \returns a reference to \c *this
+   *
+   * The vectors \a u and \c v \b must be column vectors, however they can be
+   * a adjoint expression without any overhead. Only the meaningful triangular
+   * part of the matrix is updated, the rest is left unchanged.
+   *
+   * \sa rankUpdate(const MatrixBase<DerivedU>&, Scalar)
+   */
+  template <typename DerivedU, typename DerivedV>
+  EIGEN_DEVICE_FUNC SelfAdjointView& rankUpdate(const MatrixBase<DerivedU>& u, const MatrixBase<DerivedV>& v,
+                                                const Scalar& alpha = Scalar(1));
+
+  /** Perform a symmetric rank K update of the selfadjoint matrix \c *this:
+   * \f$ this = this + \alpha ( u u^* ) \f$ where \a u is a vector or matrix.
+   *
+   * \returns a reference to \c *this
+   *
+   * Note that to perform \f$ this = this + \alpha ( u^* u ) \f$ you can simply
+   * call this function with u.adjoint().
+   *
+   * \sa rankUpdate(const MatrixBase<DerivedU>&, const MatrixBase<DerivedV>&, Scalar)
+   */
+  template <typename DerivedU>
+  EIGEN_DEVICE_FUNC SelfAdjointView& rankUpdate(const MatrixBase<DerivedU>& u, const Scalar& alpha = Scalar(1));
+
+  /** \returns an expression of a triangular view extracted from the current selfadjoint view of a given triangular part
+   *
+   * The parameter \a TriMode can have the following values: \c #Upper, \c #StrictlyUpper, \c #UnitUpper,
+   * \c #Lower, \c #StrictlyLower, \c #UnitLower.
+   *
+   * If \c TriMode references the same triangular part than \c *this, then this method simply return a \c TriangularView
+   * of the nested expression, otherwise, the nested expression is first transposed, thus returning a \c
+   * TriangularView<Transpose<MatrixType>> object.
+   *
+   * \sa MatrixBase::triangularView(), class TriangularView
+   */
+  template <unsigned int TriMode>
+  EIGEN_DEVICE_FUNC
+      std::conditional_t<(TriMode & (Upper | Lower)) == (UpLo & (Upper | Lower)), TriangularView<MatrixType, TriMode>,
+                         TriangularView<typename MatrixType::AdjointReturnType, TriMode> >
+      triangularView() const {
+    std::conditional_t<(TriMode & (Upper | Lower)) == (UpLo & (Upper | Lower)), MatrixType&,
+                       typename MatrixType::ConstTransposeReturnType>
+        tmp1(m_matrix);
+    std::conditional_t<(TriMode & (Upper | Lower)) == (UpLo & (Upper | Lower)), MatrixType&,
+                       typename MatrixType::AdjointReturnType>
+        tmp2(tmp1);
+    return std::conditional_t<(TriMode & (Upper | Lower)) == (UpLo & (Upper | Lower)),
+                              TriangularView<MatrixType, TriMode>,
+                              TriangularView<typename MatrixType::AdjointReturnType, TriMode> >(tmp2);
+  }
+
+  typedef SelfAdjointView<const MatrixConjugateReturnType, UpLo> ConjugateReturnType;
+  /** \sa MatrixBase::conjugate() const */
+  EIGEN_DEVICE_FUNC inline const ConjugateReturnType conjugate() const {
+    return ConjugateReturnType(m_matrix.conjugate());
+  }
+
+  /** \returns an expression of the complex conjugate of \c *this if Cond==true,
+   *           returns \c *this otherwise.
+   */
+  template <bool Cond>
+  EIGEN_DEVICE_FUNC inline std::conditional_t<Cond, ConjugateReturnType, ConstSelfAdjointView> conjugateIf() const {
+    typedef std::conditional_t<Cond, ConjugateReturnType, ConstSelfAdjointView> ReturnType;
+    return ReturnType(m_matrix.template conjugateIf<Cond>());
+  }
+
+  typedef SelfAdjointView<const typename MatrixType::AdjointReturnType, TransposeMode> AdjointReturnType;
+  /** \sa MatrixBase::adjoint() const */
+  EIGEN_DEVICE_FUNC inline const AdjointReturnType adjoint() const { return AdjointReturnType(m_matrix.adjoint()); }
+
+  typedef SelfAdjointView<typename MatrixType::TransposeReturnType, TransposeMode> TransposeReturnType;
+  /** \sa MatrixBase::transpose() */
+  template <class Dummy = int>
+  EIGEN_DEVICE_FUNC inline TransposeReturnType transpose(
+      std::enable_if_t<Eigen::internal::is_lvalue<MatrixType>::value, Dummy*> = nullptr) {
+    typename MatrixType::TransposeReturnType tmp(m_matrix);
+    return TransposeReturnType(tmp);
+  }
+
+  typedef SelfAdjointView<const typename MatrixType::ConstTransposeReturnType, TransposeMode> ConstTransposeReturnType;
+  /** \sa MatrixBase::transpose() const */
+  EIGEN_DEVICE_FUNC inline const ConstTransposeReturnType transpose() const {
+    return ConstTransposeReturnType(m_matrix.transpose());
+  }
+
+  /** \returns a const expression of the main diagonal of the matrix \c *this
+   *
+   * This method simply returns the diagonal of the nested expression, thus by-passing the SelfAdjointView decorator.
+   *
+   * \sa MatrixBase::diagonal(), class Diagonal */
+  EIGEN_DEVICE_FUNC typename MatrixType::ConstDiagonalReturnType diagonal() const {
+    return typename MatrixType::ConstDiagonalReturnType(m_matrix);
+  }
+
+  /////////// Cholesky module ///////////
+
+  const LLT<PlainObject, UpLo> llt() const;
+  const LDLT<PlainObject, UpLo> ldlt() const;
+
+  /////////// Eigenvalue module ///////////
+
+  /** Real part of #Scalar */
+  typedef typename NumTraits<Scalar>::Real RealScalar;
+  /** Return type of eigenvalues() */
+  typedef Matrix<RealScalar, internal::traits<MatrixType>::ColsAtCompileTime, 1> EigenvaluesReturnType;
+
+  EIGEN_DEVICE_FUNC EigenvaluesReturnType eigenvalues() const;
+  EIGEN_DEVICE_FUNC RealScalar operatorNorm() const;
+
+ protected:
+  MatrixTypeNested m_matrix;
+};
+
+// template<typename OtherDerived, typename MatrixType, unsigned int UpLo>
+// internal::selfadjoint_matrix_product_returntype<OtherDerived,SelfAdjointView<MatrixType,UpLo> >
+// operator*(const MatrixBase<OtherDerived>& lhs, const SelfAdjointView<MatrixType,UpLo>& rhs)
+// {
+//   return internal::matrix_selfadjoint_product_returntype<OtherDerived,SelfAdjointView<MatrixType,UpLo>
+//   >(lhs.derived(),rhs);
+// }
+
+// selfadjoint to dense matrix
+
+namespace internal {
+
+// TODO currently a selfadjoint expression has the form SelfAdjointView<.,.>
+//      in the future selfadjoint-ness should be defined by the expression traits
+//      such that Transpose<SelfAdjointView<.,.> > is valid. (currently TriangularBase::transpose() is overloaded to
+//      make it work)
+template <typename MatrixType, unsigned int Mode>
+struct evaluator_traits<SelfAdjointView<MatrixType, Mode> > {
+  typedef typename storage_kind_to_evaluator_kind<typename MatrixType::StorageKind>::Kind Kind;
+  typedef SelfAdjointShape Shape;
+};
+
+template <int UpLo, int SetOpposite, typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor,
+          int Version>
+class triangular_dense_assignment_kernel<UpLo, SelfAdjoint, SetOpposite, DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor,
+                                         Version>
+    : public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, Version> {
+ protected:
+  typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, Version> Base;
+  typedef typename Base::DstXprType DstXprType;
+  typedef typename Base::SrcXprType SrcXprType;
+  using Base::m_dst;
+  using Base::m_functor;
+  using Base::m_src;
+
+ public:
+  typedef typename Base::DstEvaluatorType DstEvaluatorType;
+  typedef typename Base::SrcEvaluatorType SrcEvaluatorType;
+  typedef typename Base::Scalar Scalar;
+  typedef typename Base::AssignmentTraits AssignmentTraits;
+
+  EIGEN_DEVICE_FUNC triangular_dense_assignment_kernel(DstEvaluatorType& dst, const SrcEvaluatorType& src,
+                                                       const Functor& func, DstXprType& dstExpr)
+      : Base(dst, src, func, dstExpr) {}
+
+  EIGEN_DEVICE_FUNC void assignCoeff(Index row, Index col) {
+    eigen_internal_assert(row != col);
+    Scalar tmp = m_src.coeff(row, col);
+    m_functor.assignCoeff(m_dst.coeffRef(row, col), tmp);
+    m_functor.assignCoeff(m_dst.coeffRef(col, row), numext::conj(tmp));
+  }
+
+  EIGEN_DEVICE_FUNC void assignDiagonalCoeff(Index id) { Base::assignCoeff(id, id); }
+
+  EIGEN_DEVICE_FUNC void assignOppositeCoeff(Index, Index) { eigen_internal_assert(false && "should never be called"); }
+};
+
+}  // end namespace internal
+
+/***************************************************************************
+ * Implementation of MatrixBase methods
+ ***************************************************************************/
+
+/** This is the const version of MatrixBase::selfadjointView() */
+template <typename Derived>
+template <unsigned int UpLo>
+EIGEN_DEVICE_FUNC typename MatrixBase<Derived>::template ConstSelfAdjointViewReturnType<UpLo>::Type
+MatrixBase<Derived>::selfadjointView() const {
+  return typename ConstSelfAdjointViewReturnType<UpLo>::Type(derived());
+}
+
+/** \returns an expression of a symmetric/self-adjoint view extracted from the upper or lower triangular part of the
+ * current matrix
+ *
+ * The parameter \a UpLo can be either \c #Upper or \c #Lower
+ *
+ * Example: \include MatrixBase_selfadjointView.cpp
+ * Output: \verbinclude MatrixBase_selfadjointView.out
+ *
+ * \sa class SelfAdjointView
+ */
+template <typename Derived>
+template <unsigned int UpLo>
+EIGEN_DEVICE_FUNC typename MatrixBase<Derived>::template SelfAdjointViewReturnType<UpLo>::Type
+MatrixBase<Derived>::selfadjointView() {
+  return typename SelfAdjointViewReturnType<UpLo>::Type(derived());
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_SELFADJOINTMATRIX_H

include/Eigen/src/Core/SelfCwiseBinaryOp.h

@@ -0,0 +1,50 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_SELFCWISEBINARYOP_H
+#define EIGEN_SELFCWISEBINARYOP_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+// TODO generalize the scalar type of 'other'
+
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator*=(const Scalar& other) {
+  internal::call_assignment(this->derived(), PlainObject::Constant(rows(), cols(), other),
+                            internal::mul_assign_op<Scalar, Scalar>());
+  return derived();
+}
+
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase<Derived>::operator+=(const Scalar& other) {
+  internal::call_assignment(this->derived(), PlainObject::Constant(rows(), cols(), other),
+                            internal::add_assign_op<Scalar, Scalar>());
+  return derived();
+}
+
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase<Derived>::operator-=(const Scalar& other) {
+  internal::call_assignment(this->derived(), PlainObject::Constant(rows(), cols(), other),
+                            internal::sub_assign_op<Scalar, Scalar>());
+  return derived();
+}
+
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator/=(const Scalar& other) {
+  internal::call_assignment(this->derived(), PlainObject::Constant(rows(), cols(), other),
+                            internal::div_assign_op<Scalar, Scalar>());
+  return derived();
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_SELFCWISEBINARYOP_H

include/Eigen/src/Core/SkewSymmetricMatrix3.h

@@ -0,0 +1,382 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2007-2009 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_SKEWSYMMETRICMATRIX3_H
+#define EIGEN_SKEWSYMMETRICMATRIX3_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class SkewSymmetricBase
+ * \ingroup Core_Module
+ *
+ * \brief Base class for skew symmetric matrices and expressions
+ *
+ * This is the base class that is inherited by SkewSymmetricMatrix3 and related expression
+ * types, which internally use a three vector for storing the entries. SkewSymmetric
+ * types always represent square three times three matrices.
+ *
+ * This implementations follows class DiagonalMatrix
+ *
+ * \tparam Derived is the derived type, a SkewSymmetricMatrix3 or SkewSymmetricWrapper.
+ *
+ * \sa class SkewSymmetricMatrix3, class SkewSymmetricWrapper
+ */
+template <typename Derived>
+class SkewSymmetricBase : public EigenBase<Derived> {
+ public:
+  typedef typename internal::traits<Derived>::SkewSymmetricVectorType SkewSymmetricVectorType;
+  typedef typename SkewSymmetricVectorType::Scalar Scalar;
+  typedef typename SkewSymmetricVectorType::RealScalar RealScalar;
+  typedef typename internal::traits<Derived>::StorageKind StorageKind;
+  typedef typename internal::traits<Derived>::StorageIndex StorageIndex;
+
+  enum {
+    RowsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime,
+    ColsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime,
+    MaxRowsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime,
+    MaxColsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime,
+    IsVectorAtCompileTime = 0,
+    Flags = NoPreferredStorageOrderBit
+  };
+
+  typedef Matrix<Scalar, RowsAtCompileTime, ColsAtCompileTime, 0, MaxRowsAtCompileTime, MaxColsAtCompileTime>
+      DenseMatrixType;
+  typedef DenseMatrixType DenseType;
+  typedef SkewSymmetricMatrix3<Scalar> PlainObject;
+
+  /** \returns a reference to the derived object. */
+  EIGEN_DEVICE_FUNC inline const Derived& derived() const { return *static_cast<const Derived*>(this); }
+  /** \returns a const reference to the derived object. */
+  EIGEN_DEVICE_FUNC inline Derived& derived() { return *static_cast<Derived*>(this); }
+
+  /**
+   * Constructs a dense matrix from \c *this. Note, this directly returns a dense matrix type,
+   * not an expression.
+   * \returns A dense matrix, with its entries set from the the derived object. */
+  EIGEN_DEVICE_FUNC DenseMatrixType toDenseMatrix() const { return derived(); }
+
+  /** Determinant vanishes */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Scalar determinant() const { return 0; }
+
+  /** A.transpose() = -A */
+  EIGEN_DEVICE_FUNC PlainObject transpose() const { return (-vector()).asSkewSymmetric(); }
+
+  /** \returns the exponential of this matrix using Rodrigues’ formula */
+  EIGEN_DEVICE_FUNC DenseMatrixType exponential() const {
+    DenseMatrixType retVal = DenseMatrixType::Identity();
+    const SkewSymmetricVectorType& v = vector();
+    if (v.isZero()) {
+      return retVal;
+    }
+    const Scalar norm2 = v.squaredNorm();
+    const Scalar norm = numext::sqrt(norm2);
+    retVal += ((((1 - numext::cos(norm)) / norm2) * derived()) * derived()) +
+              (numext::sin(norm) / norm) * derived().toDenseMatrix();
+    return retVal;
+  }
+
+  /** \returns a reference to the derived object's vector of coefficients. */
+  EIGEN_DEVICE_FUNC inline const SkewSymmetricVectorType& vector() const { return derived().vector(); }
+  /** \returns a const reference to the derived object's vector of coefficients. */
+  EIGEN_DEVICE_FUNC inline SkewSymmetricVectorType& vector() { return derived().vector(); }
+
+  /** \returns the number of rows. */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const { return 3; }
+  /** \returns the number of columns. */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const { return 3; }
+
+  /** \returns the matrix product of \c *this by the dense matrix, \a matrix */
+  template <typename MatrixDerived>
+  EIGEN_DEVICE_FUNC Product<Derived, MatrixDerived, LazyProduct> operator*(
+      const MatrixBase<MatrixDerived>& matrix) const {
+    return Product<Derived, MatrixDerived, LazyProduct>(derived(), matrix.derived());
+  }
+
+  /** \returns the matrix product of \c *this by the skew symmetric matrix, \a matrix */
+  template <typename MatrixDerived>
+  EIGEN_DEVICE_FUNC Product<Derived, MatrixDerived, LazyProduct> operator*(
+      const SkewSymmetricBase<MatrixDerived>& matrix) const {
+    return Product<Derived, MatrixDerived, LazyProduct>(derived(), matrix.derived());
+  }
+
+  template <typename OtherDerived>
+  using SkewSymmetricProductReturnType = SkewSymmetricWrapper<const EIGEN_CWISE_BINARY_RETURN_TYPE(
+      SkewSymmetricVectorType, typename OtherDerived::SkewSymmetricVectorType, product)>;
+
+  /** \returns the wedge product of \c *this by the skew symmetric matrix \a other
+   *  A wedge B = AB - BA */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC SkewSymmetricProductReturnType<OtherDerived> wedge(
+      const SkewSymmetricBase<OtherDerived>& other) const {
+    return vector().cross(other.vector()).asSkewSymmetric();
+  }
+
+  using SkewSymmetricScaleReturnType =
+      SkewSymmetricWrapper<const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(SkewSymmetricVectorType, Scalar, product)>;
+
+  /** \returns the product of \c *this by the scalar \a scalar */
+  EIGEN_DEVICE_FUNC inline SkewSymmetricScaleReturnType operator*(const Scalar& scalar) const {
+    return (vector() * scalar).asSkewSymmetric();
+  }
+
+  using ScaleSkewSymmetricReturnType =
+      SkewSymmetricWrapper<const EIGEN_SCALAR_BINARYOP_EXPR_RETURN_TYPE(Scalar, SkewSymmetricVectorType, product)>;
+
+  /** \returns the product of a scalar and the skew symmetric matrix \a other */
+  EIGEN_DEVICE_FUNC friend inline ScaleSkewSymmetricReturnType operator*(const Scalar& scalar,
+                                                                         const SkewSymmetricBase& other) {
+    return (scalar * other.vector()).asSkewSymmetric();
+  }
+
+  template <typename OtherDerived>
+  using SkewSymmetricSumReturnType = SkewSymmetricWrapper<const EIGEN_CWISE_BINARY_RETURN_TYPE(
+      SkewSymmetricVectorType, typename OtherDerived::SkewSymmetricVectorType, sum)>;
+
+  /** \returns the sum of \c *this and the skew symmetric matrix \a other */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline SkewSymmetricSumReturnType<OtherDerived> operator+(
+      const SkewSymmetricBase<OtherDerived>& other) const {
+    return (vector() + other.vector()).asSkewSymmetric();
+  }
+
+  template <typename OtherDerived>
+  using SkewSymmetricDifferenceReturnType = SkewSymmetricWrapper<const EIGEN_CWISE_BINARY_RETURN_TYPE(
+      SkewSymmetricVectorType, typename OtherDerived::SkewSymmetricVectorType, difference)>;
+
+  /** \returns the difference of \c *this and the skew symmetric matrix \a other */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline SkewSymmetricDifferenceReturnType<OtherDerived> operator-(
+      const SkewSymmetricBase<OtherDerived>& other) const {
+    return (vector() - other.vector()).asSkewSymmetric();
+  }
+};
+
+/** \class SkewSymmetricMatrix3
+ * \ingroup Core_Module
+ *
+ * \brief Represents a 3x3 skew symmetric matrix with its storage
+ *
+ * \tparam Scalar_ the type of coefficients
+ *
+ * \sa class SkewSymmetricBase, class SkewSymmetricWrapper
+ */
+
+namespace internal {
+template <typename Scalar_>
+struct traits<SkewSymmetricMatrix3<Scalar_>> : traits<Matrix<Scalar_, 3, 3, 0, 3, 3>> {
+  typedef Matrix<Scalar_, 3, 1, 0, 3, 1> SkewSymmetricVectorType;
+  typedef SkewSymmetricShape StorageKind;
+  enum { Flags = LvalueBit | NoPreferredStorageOrderBit | NestByRefBit };
+};
+}  // namespace internal
+template <typename Scalar_>
+class SkewSymmetricMatrix3 : public SkewSymmetricBase<SkewSymmetricMatrix3<Scalar_>> {
+ public:
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef typename internal::traits<SkewSymmetricMatrix3>::SkewSymmetricVectorType SkewSymmetricVectorType;
+  typedef const SkewSymmetricMatrix3& Nested;
+  typedef Scalar_ Scalar;
+  typedef typename internal::traits<SkewSymmetricMatrix3>::StorageKind StorageKind;
+  typedef typename internal::traits<SkewSymmetricMatrix3>::StorageIndex StorageIndex;
+#endif
+
+ protected:
+  SkewSymmetricVectorType m_vector;
+
+ public:
+  /** const version of vector(). */
+  EIGEN_DEVICE_FUNC inline const SkewSymmetricVectorType& vector() const { return m_vector; }
+  /** \returns a reference to the stored vector of coefficients. */
+  EIGEN_DEVICE_FUNC inline SkewSymmetricVectorType& vector() { return m_vector; }
+
+  /** Default constructor without initialization */
+  EIGEN_DEVICE_FUNC inline SkewSymmetricMatrix3() {}
+
+  /** Constructor from three scalars */
+  EIGEN_DEVICE_FUNC inline SkewSymmetricMatrix3(const Scalar& x, const Scalar& y, const Scalar& z)
+      : m_vector(x, y, z) {}
+
+  /** \brief Constructs a SkewSymmetricMatrix3 from an r-value vector type */
+  EIGEN_DEVICE_FUNC explicit inline SkewSymmetricMatrix3(SkewSymmetricVectorType&& vec) : m_vector(std::move(vec)) {}
+
+  /** generic constructor from expression of the coefficients */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC explicit inline SkewSymmetricMatrix3(const MatrixBase<OtherDerived>& other) : m_vector(other) {}
+
+  /** Copy constructor. */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC inline SkewSymmetricMatrix3(const SkewSymmetricBase<OtherDerived>& other)
+      : m_vector(other.vector()) {}
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** copy constructor. prevent a default copy constructor from hiding the other templated constructor */
+  inline SkewSymmetricMatrix3(const SkewSymmetricMatrix3& other) : m_vector(other.vector()) {}
+#endif
+
+  /** Copy operator. */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC SkewSymmetricMatrix3& operator=(const SkewSymmetricBase<OtherDerived>& other) {
+    m_vector = other.vector();
+    return *this;
+  }
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** This is a special case of the templated operator=. Its purpose is to
+   * prevent a default operator= from hiding the templated operator=.
+   */
+  EIGEN_DEVICE_FUNC SkewSymmetricMatrix3& operator=(const SkewSymmetricMatrix3& other) {
+    m_vector = other.vector();
+    return *this;
+  }
+#endif
+
+  typedef SkewSymmetricWrapper<const CwiseNullaryOp<internal::scalar_constant_op<Scalar>, SkewSymmetricVectorType>>
+      InitializeReturnType;
+
+  /** Initializes a skew symmetric matrix with coefficients set to zero */
+  EIGEN_DEVICE_FUNC static InitializeReturnType Zero() { return SkewSymmetricVectorType::Zero().asSkewSymmetric(); }
+
+  /** Sets all coefficients to zero. */
+  EIGEN_DEVICE_FUNC inline void setZero() { m_vector.setZero(); }
+};
+
+/** \class SkewSymmetricWrapper
+ * \ingroup Core_Module
+ *
+ * \brief Expression of a skew symmetric matrix
+ *
+ * \tparam SkewSymmetricVectorType_ the type of the vector of coefficients
+ *
+ * This class is an expression of a skew symmetric matrix, but not storing its own vector of coefficients,
+ * instead wrapping an existing vector expression. It is the return type of MatrixBase::asSkewSymmetric()
+ * and most of the time this is the only way that it is used.
+ *
+ * \sa class SkewSymmetricMatrix3, class SkewSymmetricBase, MatrixBase::asSkewSymmetric()
+ */
+
+namespace internal {
+template <typename SkewSymmetricVectorType_>
+struct traits<SkewSymmetricWrapper<SkewSymmetricVectorType_>> {
+  typedef SkewSymmetricVectorType_ SkewSymmetricVectorType;
+  typedef typename SkewSymmetricVectorType::Scalar Scalar;
+  typedef typename SkewSymmetricVectorType::StorageIndex StorageIndex;
+  typedef SkewSymmetricShape StorageKind;
+  typedef typename traits<SkewSymmetricVectorType>::XprKind XprKind;
+  enum {
+    RowsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime,
+    ColsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime,
+    MaxRowsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime,
+    MaxColsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime,
+    Flags = (traits<SkewSymmetricVectorType>::Flags & LvalueBit) | NoPreferredStorageOrderBit
+  };
+};
+}  // namespace internal
+
+template <typename SkewSymmetricVectorType_>
+class SkewSymmetricWrapper : public SkewSymmetricBase<SkewSymmetricWrapper<SkewSymmetricVectorType_>>,
+                             internal::no_assignment_operator {
+ public:
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  typedef SkewSymmetricVectorType_ SkewSymmetricVectorType;
+  typedef SkewSymmetricWrapper Nested;
+#endif
+
+  /** Constructor from expression of coefficients to wrap. */
+  EIGEN_DEVICE_FUNC explicit inline SkewSymmetricWrapper(SkewSymmetricVectorType& a_vector) : m_vector(a_vector) {}
+
+  /** \returns a const reference to the wrapped expression of coefficients. */
+  EIGEN_DEVICE_FUNC const SkewSymmetricVectorType& vector() const { return m_vector; }
+
+ protected:
+  typename SkewSymmetricVectorType::Nested m_vector;
+};
+
+/** \returns a pseudo-expression of a skew symmetric matrix with *this as vector of coefficients
+ *
+ * \only_for_vectors
+ *
+ * \sa class SkewSymmetricWrapper, class SkewSymmetricMatrix3, vector(), isSkewSymmetric()
+ **/
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline const SkewSymmetricWrapper<const Derived> MatrixBase<Derived>::asSkewSymmetric() const {
+  return SkewSymmetricWrapper<const Derived>(derived());
+}
+
+/** \returns true if *this is approximately equal to a skew symmetric matrix,
+ *          within the precision given by \a prec.
+ */
+template <typename Derived>
+bool MatrixBase<Derived>::isSkewSymmetric(const RealScalar& prec) const {
+  if (cols() != rows()) return false;
+  return (this->transpose() + *this).isZero(prec);
+}
+
+/** \returns the matrix product of \c *this by the skew symmetric matrix \skew.
+ */
+template <typename Derived>
+template <typename SkewDerived>
+EIGEN_DEVICE_FUNC inline const Product<Derived, SkewDerived, LazyProduct> MatrixBase<Derived>::operator*(
+    const SkewSymmetricBase<SkewDerived>& skew) const {
+  return Product<Derived, SkewDerived, LazyProduct>(derived(), skew.derived());
+}
+
+namespace internal {
+
+template <>
+struct storage_kind_to_shape<SkewSymmetricShape> {
+  typedef SkewSymmetricShape Shape;
+};
+
+struct SkewSymmetric2Dense {};
+
+template <>
+struct AssignmentKind<DenseShape, SkewSymmetricShape> {
+  typedef SkewSymmetric2Dense Kind;
+};
+
+// SkewSymmetric matrix to Dense assignment
+template <typename DstXprType, typename SrcXprType, typename Functor>
+struct Assignment<DstXprType, SrcXprType, Functor, SkewSymmetric2Dense> {
+  EIGEN_DEVICE_FUNC static void run(
+      DstXprType& dst, const SrcXprType& src,
+      const internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>& /*func*/) {
+    if ((dst.rows() != 3) || (dst.cols() != 3)) {
+      dst.resize(3, 3);
+    }
+    dst.diagonal().setZero();
+    const typename SrcXprType::SkewSymmetricVectorType v = src.vector();
+    dst(0, 1) = -v(2);
+    dst(1, 0) = v(2);
+    dst(0, 2) = v(1);
+    dst(2, 0) = -v(1);
+    dst(1, 2) = -v(0);
+    dst(2, 1) = v(0);
+  }
+  EIGEN_DEVICE_FUNC static void run(
+      DstXprType& dst, const SrcXprType& src,
+      const internal::add_assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>& /*func*/) {
+    dst.vector() += src.vector();
+  }
+
+  EIGEN_DEVICE_FUNC static void run(
+      DstXprType& dst, const SrcXprType& src,
+      const internal::sub_assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>& /*func*/) {
+    dst.vector() -= src.vector();
+  }
+};
+
+}  // namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_SKEWSYMMETRICMATRIX3_H

include/Eigen/src/Core/Solve.h

@@ -0,0 +1,174 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2014 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_SOLVE_H
+#define EIGEN_SOLVE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+template <typename Decomposition, typename RhsType, typename StorageKind>
+class SolveImpl;
+
+/** \class Solve
+ * \ingroup Core_Module
+ *
+ * \brief Pseudo expression representing a solving operation
+ *
+ * \tparam Decomposition the type of the matrix or decomposition object
+ * \tparam Rhstype the type of the right-hand side
+ *
+ * This class represents an expression of A.solve(B)
+ * and most of the time this is the only way it is used.
+ *
+ */
+namespace internal {
+
+// this solve_traits class permits to determine the evaluation type with respect to storage kind (Dense vs Sparse)
+template <typename Decomposition, typename RhsType, typename StorageKind>
+struct solve_traits;
+
+template <typename Decomposition, typename RhsType>
+struct solve_traits<Decomposition, RhsType, Dense> {
+  typedef typename make_proper_matrix_type<typename RhsType::Scalar, Decomposition::ColsAtCompileTime,
+                                           RhsType::ColsAtCompileTime, RhsType::PlainObject::Options,
+                                           Decomposition::MaxColsAtCompileTime, RhsType::MaxColsAtCompileTime>::type
+      PlainObject;
+};
+
+template <typename Decomposition, typename RhsType>
+struct traits<Solve<Decomposition, RhsType> >
+    : traits<
+          typename solve_traits<Decomposition, RhsType, typename internal::traits<RhsType>::StorageKind>::PlainObject> {
+  typedef typename solve_traits<Decomposition, RhsType, typename internal::traits<RhsType>::StorageKind>::PlainObject
+      PlainObject;
+  typedef typename promote_index_type<typename Decomposition::StorageIndex, typename RhsType::StorageIndex>::type
+      StorageIndex;
+  typedef traits<PlainObject> BaseTraits;
+  enum { Flags = BaseTraits::Flags & RowMajorBit, CoeffReadCost = HugeCost };
+};
+
+}  // namespace internal
+
+template <typename Decomposition, typename RhsType>
+class Solve : public SolveImpl<Decomposition, RhsType, typename internal::traits<RhsType>::StorageKind> {
+ public:
+  typedef typename internal::traits<Solve>::PlainObject PlainObject;
+  typedef typename internal::traits<Solve>::StorageIndex StorageIndex;
+
+  Solve(const Decomposition &dec, const RhsType &rhs) : m_dec(dec), m_rhs(rhs) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_dec.cols(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_rhs.cols(); }
+
+  EIGEN_DEVICE_FUNC const Decomposition &dec() const { return m_dec; }
+  EIGEN_DEVICE_FUNC const RhsType &rhs() const { return m_rhs; }
+
+ protected:
+  const Decomposition &m_dec;
+  const typename internal::ref_selector<RhsType>::type m_rhs;
+};
+
+// Specialization of the Solve expression for dense results
+template <typename Decomposition, typename RhsType>
+class SolveImpl<Decomposition, RhsType, Dense> : public MatrixBase<Solve<Decomposition, RhsType> > {
+  typedef Solve<Decomposition, RhsType> Derived;
+
+ public:
+  typedef MatrixBase<Solve<Decomposition, RhsType> > Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Derived)
+
+ private:
+  Scalar coeff(Index row, Index col) const;
+  Scalar coeff(Index i) const;
+};
+
+// Generic API dispatcher
+template <typename Decomposition, typename RhsType, typename StorageKind>
+class SolveImpl : public internal::generic_xpr_base<Solve<Decomposition, RhsType>, MatrixXpr, StorageKind>::type {
+ public:
+  typedef typename internal::generic_xpr_base<Solve<Decomposition, RhsType>, MatrixXpr, StorageKind>::type Base;
+};
+
+namespace internal {
+
+// Evaluator of Solve -> eval into a temporary
+template <typename Decomposition, typename RhsType>
+struct evaluator<Solve<Decomposition, RhsType> >
+    : public evaluator<typename Solve<Decomposition, RhsType>::PlainObject> {
+  typedef Solve<Decomposition, RhsType> SolveType;
+  typedef typename SolveType::PlainObject PlainObject;
+  typedef evaluator<PlainObject> Base;
+
+  enum { Flags = Base::Flags | EvalBeforeNestingBit };
+
+  EIGEN_DEVICE_FUNC explicit evaluator(const SolveType &solve) : m_result(solve.rows(), solve.cols()) {
+    internal::construct_at<Base>(this, m_result);
+    solve.dec()._solve_impl(solve.rhs(), m_result);
+  }
+
+ protected:
+  PlainObject m_result;
+};
+
+// Specialization for "dst = dec.solve(rhs)"
+// NOTE we need to specialize it for Dense2Dense to avoid ambiguous specialization error and a Sparse2Sparse
+// specialization must exist somewhere
+template <typename DstXprType, typename DecType, typename RhsType, typename Scalar>
+struct Assignment<DstXprType, Solve<DecType, RhsType>, internal::assign_op<Scalar, Scalar>, Dense2Dense> {
+  typedef Solve<DecType, RhsType> SrcXprType;
+  static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar, Scalar> &) {
+    Index dstRows = src.rows();
+    Index dstCols = src.cols();
+    if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
+
+    src.dec()._solve_impl(src.rhs(), dst);
+  }
+};
+
+// Specialization for "dst = dec.transpose().solve(rhs)"
+template <typename DstXprType, typename DecType, typename RhsType, typename Scalar>
+struct Assignment<DstXprType, Solve<Transpose<const DecType>, RhsType>, internal::assign_op<Scalar, Scalar>,
+                  Dense2Dense> {
+  typedef Solve<Transpose<const DecType>, RhsType> SrcXprType;
+  static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar, Scalar> &) {
+    Index dstRows = src.rows();
+    Index dstCols = src.cols();
+    if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
+
+    src.dec().nestedExpression().template _solve_impl_transposed<false>(src.rhs(), dst);
+  }
+};
+
+// Specialization for "dst = dec.adjoint().solve(rhs)"
+template <typename DstXprType, typename DecType, typename RhsType, typename Scalar>
+struct Assignment<
+    DstXprType,
+    Solve<CwiseUnaryOp<internal::scalar_conjugate_op<typename DecType::Scalar>, const Transpose<const DecType> >,
+          RhsType>,
+    internal::assign_op<Scalar, Scalar>, Dense2Dense> {
+  typedef Solve<CwiseUnaryOp<internal::scalar_conjugate_op<typename DecType::Scalar>, const Transpose<const DecType> >,
+                RhsType>
+      SrcXprType;
+  static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar, Scalar> &) {
+    Index dstRows = src.rows();
+    Index dstCols = src.cols();
+    if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
+
+    src.dec().nestedExpression().nestedExpression().template _solve_impl_transposed<true>(src.rhs(), dst);
+  }
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_SOLVE_H

include/Eigen/src/Core/SolveTriangular.h

@@ -0,0 +1,237 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_SOLVETRIANGULAR_H
+#define EIGEN_SOLVETRIANGULAR_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+// Forward declarations:
+// The following two routines are implemented in the products/TriangularSolver*.h files
+template <typename LhsScalar, typename RhsScalar, typename Index, int Side, int Mode, bool Conjugate, int StorageOrder>
+struct triangular_solve_vector;
+
+template <typename Scalar, typename Index, int Side, int Mode, bool Conjugate, int TriStorageOrder,
+          int OtherStorageOrder, int OtherInnerStride>
+struct triangular_solve_matrix;
+
+// small helper struct extracting some traits on the underlying solver operation
+template <typename Lhs, typename Rhs, int Side>
+class trsolve_traits {
+ private:
+  enum { RhsIsVectorAtCompileTime = (Side == OnTheLeft ? Rhs::ColsAtCompileTime : Rhs::RowsAtCompileTime) == 1 };
+
+ public:
+  enum {
+    Unrolling = (RhsIsVectorAtCompileTime && Rhs::SizeAtCompileTime != Dynamic && Rhs::SizeAtCompileTime <= 8)
+                    ? CompleteUnrolling
+                    : NoUnrolling,
+    RhsVectors = RhsIsVectorAtCompileTime ? 1 : Dynamic
+  };
+};
+
+template <typename Lhs, typename Rhs,
+          int Side,  // can be OnTheLeft/OnTheRight
+          int Mode,  // can be Upper/Lower | UnitDiag
+          int Unrolling = trsolve_traits<Lhs, Rhs, Side>::Unrolling,
+          int RhsVectors = trsolve_traits<Lhs, Rhs, Side>::RhsVectors>
+struct triangular_solver_selector;
+
+template <typename Lhs, typename Rhs, int Side, int Mode>
+struct triangular_solver_selector<Lhs, Rhs, Side, Mode, NoUnrolling, 1> {
+  typedef typename Lhs::Scalar LhsScalar;
+  typedef typename Rhs::Scalar RhsScalar;
+  typedef blas_traits<Lhs> LhsProductTraits;
+  typedef typename LhsProductTraits::ExtractType ActualLhsType;
+  typedef Map<Matrix<RhsScalar, Dynamic, 1>, Aligned> MappedRhs;
+  static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) {
+    ActualLhsType actualLhs = LhsProductTraits::extract(lhs);
+
+    // FIXME find a way to allow an inner stride if packet_traits<Scalar>::size==1
+
+    bool useRhsDirectly = Rhs::InnerStrideAtCompileTime == 1 || rhs.innerStride() == 1;
+
+    ei_declare_aligned_stack_constructed_variable(RhsScalar, actualRhs, rhs.size(), (useRhsDirectly ? rhs.data() : 0));
+
+    if (!useRhsDirectly) MappedRhs(actualRhs, rhs.size()) = rhs;
+
+    triangular_solve_vector<LhsScalar, RhsScalar, Index, Side, Mode, LhsProductTraits::NeedToConjugate,
+                            (int(Lhs::Flags) & RowMajorBit) ? RowMajor : ColMajor>::run(actualLhs.cols(),
+                                                                                        actualLhs.data(),
+                                                                                        actualLhs.outerStride(),
+                                                                                        actualRhs);
+
+    if (!useRhsDirectly) rhs = MappedRhs(actualRhs, rhs.size());
+  }
+};
+
+// the rhs is a matrix
+template <typename Lhs, typename Rhs, int Side, int Mode>
+struct triangular_solver_selector<Lhs, Rhs, Side, Mode, NoUnrolling, Dynamic> {
+  typedef typename Rhs::Scalar Scalar;
+  typedef blas_traits<Lhs> LhsProductTraits;
+  typedef typename LhsProductTraits::DirectLinearAccessType ActualLhsType;
+
+  static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) {
+    add_const_on_value_type_t<ActualLhsType> actualLhs = LhsProductTraits::extract(lhs);
+
+    const Index size = lhs.rows();
+    const Index othersize = Side == OnTheLeft ? rhs.cols() : rhs.rows();
+
+    typedef internal::gemm_blocking_space<(Rhs::Flags & RowMajorBit) ? RowMajor : ColMajor, Scalar, Scalar,
+                                          Rhs::MaxRowsAtCompileTime, Rhs::MaxColsAtCompileTime,
+                                          Lhs::MaxRowsAtCompileTime, 4>
+        BlockingType;
+
+    // Nothing to solve.
+    if (actualLhs.size() == 0 || rhs.size() == 0) {
+      return;
+    }
+
+    BlockingType blocking(rhs.rows(), rhs.cols(), size, 1, false);
+
+    triangular_solve_matrix<Scalar, Index, Side, Mode, LhsProductTraits::NeedToConjugate,
+                            (int(Lhs::Flags) & RowMajorBit) ? RowMajor : ColMajor,
+                            (Rhs::Flags & RowMajorBit) ? RowMajor : ColMajor,
+                            Rhs::InnerStrideAtCompileTime>::run(size, othersize, &actualLhs.coeffRef(0, 0),
+                                                                actualLhs.outerStride(), &rhs.coeffRef(0, 0),
+                                                                rhs.innerStride(), rhs.outerStride(), blocking);
+  }
+};
+
+/***************************************************************************
+ * meta-unrolling implementation
+ ***************************************************************************/
+
+template <typename Lhs, typename Rhs, int Mode, int LoopIndex, int Size, bool Stop = LoopIndex == Size>
+struct triangular_solver_unroller;
+
+template <typename Lhs, typename Rhs, int Mode, int LoopIndex, int Size>
+struct triangular_solver_unroller<Lhs, Rhs, Mode, LoopIndex, Size, false> {
+  enum {
+    IsLower = ((Mode & Lower) == Lower),
+    DiagIndex = IsLower ? LoopIndex : Size - LoopIndex - 1,
+    StartIndex = IsLower ? 0 : DiagIndex + 1
+  };
+  static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) {
+    if (LoopIndex > 0)
+      rhs.coeffRef(DiagIndex) -= lhs.row(DiagIndex)
+                                     .template segment<LoopIndex>(StartIndex)
+                                     .transpose()
+                                     .cwiseProduct(rhs.template segment<LoopIndex>(StartIndex))
+                                     .sum();
+
+    if (!(Mode & UnitDiag)) rhs.coeffRef(DiagIndex) /= lhs.coeff(DiagIndex, DiagIndex);
+
+    triangular_solver_unroller<Lhs, Rhs, Mode, LoopIndex + 1, Size>::run(lhs, rhs);
+  }
+};
+
+template <typename Lhs, typename Rhs, int Mode, int LoopIndex, int Size>
+struct triangular_solver_unroller<Lhs, Rhs, Mode, LoopIndex, Size, true> {
+  static EIGEN_DEVICE_FUNC void run(const Lhs&, Rhs&) {}
+};
+
+template <typename Lhs, typename Rhs, int Mode>
+struct triangular_solver_selector<Lhs, Rhs, OnTheLeft, Mode, CompleteUnrolling, 1> {
+  static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) {
+    triangular_solver_unroller<Lhs, Rhs, Mode, 0, Rhs::SizeAtCompileTime>::run(lhs, rhs);
+  }
+};
+
+template <typename Lhs, typename Rhs, int Mode>
+struct triangular_solver_selector<Lhs, Rhs, OnTheRight, Mode, CompleteUnrolling, 1> {
+  static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, Rhs& rhs) {
+    Transpose<const Lhs> trLhs(lhs);
+    Transpose<Rhs> trRhs(rhs);
+
+    triangular_solver_unroller<Transpose<const Lhs>, Transpose<Rhs>,
+                               ((Mode & Upper) == Upper ? Lower : Upper) | (Mode & UnitDiag), 0,
+                               Rhs::SizeAtCompileTime>::run(trLhs, trRhs);
+  }
+};
+
+}  // end namespace internal
+
+/***************************************************************************
+ * TriangularView methods
+ ***************************************************************************/
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+template <typename MatrixType, unsigned int Mode>
+template <int Side, typename OtherDerived>
+EIGEN_DEVICE_FUNC void TriangularViewImpl<MatrixType, Mode, Dense>::solveInPlace(
+    const MatrixBase<OtherDerived>& _other) const {
+  OtherDerived& other = _other.const_cast_derived();
+  eigen_assert(derived().cols() == derived().rows() && ((Side == OnTheLeft && derived().cols() == other.rows()) ||
+                                                        (Side == OnTheRight && derived().cols() == other.cols())));
+  eigen_assert((!(int(Mode) & int(ZeroDiag))) && bool(int(Mode) & (int(Upper) | int(Lower))));
+  // If solving for a 0x0 matrix, nothing to do, simply return.
+  if (derived().cols() == 0) return;
+
+  enum {
+    copy = (internal::traits<OtherDerived>::Flags & RowMajorBit) && OtherDerived::IsVectorAtCompileTime &&
+           OtherDerived::SizeAtCompileTime != 1
+  };
+  typedef std::conditional_t<copy, typename internal::plain_matrix_type_column_major<OtherDerived>::type, OtherDerived&>
+      OtherCopy;
+  OtherCopy otherCopy(other);
+
+  internal::triangular_solver_selector<MatrixType, std::remove_reference_t<OtherCopy>, Side, Mode>::run(
+      derived().nestedExpression(), otherCopy);
+
+  if (copy) other = otherCopy;
+}
+
+template <typename Derived, unsigned int Mode>
+template <int Side, typename Other>
+const internal::triangular_solve_retval<Side, TriangularView<Derived, Mode>, Other>
+TriangularViewImpl<Derived, Mode, Dense>::solve(const MatrixBase<Other>& other) const {
+  return internal::triangular_solve_retval<Side, TriangularViewType, Other>(derived(), other.derived());
+}
+#endif
+
+namespace internal {
+
+template <int Side, typename TriangularType, typename Rhs>
+struct traits<triangular_solve_retval<Side, TriangularType, Rhs> > {
+  typedef typename internal::plain_matrix_type_column_major<Rhs>::type ReturnType;
+};
+
+template <int Side, typename TriangularType, typename Rhs>
+struct triangular_solve_retval : public ReturnByValue<triangular_solve_retval<Side, TriangularType, Rhs> > {
+  typedef remove_all_t<typename Rhs::Nested> RhsNestedCleaned;
+  typedef ReturnByValue<triangular_solve_retval> Base;
+
+  triangular_solve_retval(const TriangularType& tri, const Rhs& rhs) : m_triangularMatrix(tri), m_rhs(rhs) {}
+
+  inline EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_rhs.rows(); }
+  inline EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_rhs.cols(); }
+
+  template <typename Dest>
+  inline void evalTo(Dest& dst) const {
+    if (!is_same_dense(dst, m_rhs)) dst = m_rhs;
+    m_triangularMatrix.template solveInPlace<Side>(dst);
+  }
+
+ protected:
+  const TriangularType& m_triangularMatrix;
+  typename Rhs::Nested m_rhs;
+};
+
+}  // namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_SOLVETRIANGULAR_H

include/Eigen/src/Core/SolverBase.h

@@ -0,0 +1,159 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2015 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_SOLVERBASE_H
+#define EIGEN_SOLVERBASE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Derived>
+struct solve_assertion {
+  template <bool Transpose_, typename Rhs>
+  static void run(const Derived& solver, const Rhs& b) {
+    solver.template _check_solve_assertion<Transpose_>(b);
+  }
+};
+
+template <typename Derived>
+struct solve_assertion<Transpose<Derived>> {
+  typedef Transpose<Derived> type;
+
+  template <bool Transpose_, typename Rhs>
+  static void run(const type& transpose, const Rhs& b) {
+    internal::solve_assertion<internal::remove_all_t<Derived>>::template run<true>(transpose.nestedExpression(), b);
+  }
+};
+
+template <typename Scalar, typename Derived>
+struct solve_assertion<CwiseUnaryOp<Eigen::internal::scalar_conjugate_op<Scalar>, const Transpose<Derived>>> {
+  typedef CwiseUnaryOp<Eigen::internal::scalar_conjugate_op<Scalar>, const Transpose<Derived>> type;
+
+  template <bool Transpose_, typename Rhs>
+  static void run(const type& adjoint, const Rhs& b) {
+    internal::solve_assertion<internal::remove_all_t<Transpose<Derived>>>::template run<true>(
+        adjoint.nestedExpression(), b);
+  }
+};
+}  // end namespace internal
+
+/** \class SolverBase
+ * \brief A base class for matrix decomposition and solvers
+ *
+ * \tparam Derived the actual type of the decomposition/solver.
+ *
+ * Any matrix decomposition inheriting this base class provide the following API:
+ *
+ * \code
+ * MatrixType A, b, x;
+ * DecompositionType dec(A);
+ * x = dec.solve(b);             // solve A   * x = b
+ * x = dec.transpose().solve(b); // solve A^T * x = b
+ * x = dec.adjoint().solve(b);   // solve A'  * x = b
+ * \endcode
+ *
+ * \warning Currently, any other usage of transpose() and adjoint() are not supported and will produce compilation
+ * errors.
+ *
+ * \sa class PartialPivLU, class FullPivLU, class HouseholderQR, class ColPivHouseholderQR, class FullPivHouseholderQR,
+ * class CompleteOrthogonalDecomposition, class LLT, class LDLT, class SVDBase
+ */
+template <typename Derived>
+class SolverBase : public EigenBase<Derived> {
+ public:
+  typedef EigenBase<Derived> Base;
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+  typedef Scalar CoeffReturnType;
+
+  template <typename Derived_>
+  friend struct internal::solve_assertion;
+
+  enum {
+    RowsAtCompileTime = internal::traits<Derived>::RowsAtCompileTime,
+    ColsAtCompileTime = internal::traits<Derived>::ColsAtCompileTime,
+    SizeAtCompileTime = (internal::size_of_xpr_at_compile_time<Derived>::ret),
+    MaxRowsAtCompileTime = internal::traits<Derived>::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = internal::traits<Derived>::MaxColsAtCompileTime,
+    MaxSizeAtCompileTime = internal::size_at_compile_time(internal::traits<Derived>::MaxRowsAtCompileTime,
+                                                          internal::traits<Derived>::MaxColsAtCompileTime),
+    IsVectorAtCompileTime =
+        internal::traits<Derived>::MaxRowsAtCompileTime == 1 || internal::traits<Derived>::MaxColsAtCompileTime == 1,
+    NumDimensions = int(MaxSizeAtCompileTime) == 1 ? 0
+                    : bool(IsVectorAtCompileTime)  ? 1
+                                                   : 2
+  };
+
+  /** Default constructor */
+  SolverBase() {}
+
+  ~SolverBase() {}
+
+  using Base::derived;
+
+  /** \returns an expression of the solution x of \f$ A x = b \f$ using the current decomposition of A.
+   */
+  template <typename Rhs>
+  inline const Solve<Derived, Rhs> solve(const MatrixBase<Rhs>& b) const {
+    internal::solve_assertion<internal::remove_all_t<Derived>>::template run<false>(derived(), b);
+    return Solve<Derived, Rhs>(derived(), b.derived());
+  }
+
+  /** \internal the return type of transpose() */
+  typedef Transpose<const Derived> ConstTransposeReturnType;
+  /** \returns an expression of the transposed of the factored matrix.
+   *
+   * A typical usage is to solve for the transposed problem A^T x = b:
+   * \code x = dec.transpose().solve(b); \endcode
+   *
+   * \sa adjoint(), solve()
+   */
+  inline const ConstTransposeReturnType transpose() const { return ConstTransposeReturnType(derived()); }
+
+  /** \internal the return type of adjoint() */
+  typedef std::conditional_t<NumTraits<Scalar>::IsComplex,
+                             CwiseUnaryOp<internal::scalar_conjugate_op<Scalar>, const ConstTransposeReturnType>,
+                             const ConstTransposeReturnType>
+      AdjointReturnType;
+  /** \returns an expression of the adjoint of the factored matrix
+   *
+   * A typical usage is to solve for the adjoint problem A' x = b:
+   * \code x = dec.adjoint().solve(b); \endcode
+   *
+   * For real scalar types, this function is equivalent to transpose().
+   *
+   * \sa transpose(), solve()
+   */
+  inline const AdjointReturnType adjoint() const { return AdjointReturnType(derived().transpose()); }
+
+ protected:
+  template <bool Transpose_, typename Rhs>
+  void _check_solve_assertion(const Rhs& b) const {
+    EIGEN_ONLY_USED_FOR_DEBUG(b);
+    eigen_assert(derived().m_isInitialized && "Solver is not initialized.");
+    eigen_assert((Transpose_ ? derived().cols() : derived().rows()) == b.rows() &&
+                 "SolverBase::solve(): invalid number of rows of the right hand side matrix b");
+  }
+};
+
+namespace internal {
+
+template <typename Derived>
+struct generic_xpr_base<Derived, MatrixXpr, SolverStorage> {
+  typedef SolverBase<Derived> type;
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_SOLVERBASE_H

include/Eigen/src/Core/StableNorm.h

@@ -0,0 +1,217 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_STABLENORM_H
+#define EIGEN_STABLENORM_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename ExpressionType, typename Scalar>
+inline void stable_norm_kernel(const ExpressionType& bl, Scalar& ssq, Scalar& scale, Scalar& invScale) {
+  Scalar maxCoeff = bl.cwiseAbs().maxCoeff();
+
+  if (maxCoeff > scale) {
+    ssq = ssq * numext::abs2(scale / maxCoeff);
+    Scalar tmp = Scalar(1) / maxCoeff;
+    if (tmp > NumTraits<Scalar>::highest()) {
+      invScale = NumTraits<Scalar>::highest();
+      scale = Scalar(1) / invScale;
+    } else if (maxCoeff > NumTraits<Scalar>::highest())  // we got a INF
+    {
+      invScale = Scalar(1);
+      scale = maxCoeff;
+    } else {
+      scale = maxCoeff;
+      invScale = tmp;
+    }
+  } else if (maxCoeff != maxCoeff)  // we got a NaN
+  {
+    scale = maxCoeff;
+  }
+
+  // TODO if the maxCoeff is much much smaller than the current scale,
+  // then we can neglect this sub vector
+  if (scale > Scalar(0))  // if scale==0, then bl is 0
+    ssq += (bl * invScale).squaredNorm();
+}
+
+template <typename VectorType, typename RealScalar>
+void stable_norm_impl_inner_step(const VectorType& vec, RealScalar& ssq, RealScalar& scale, RealScalar& invScale) {
+  const Index blockSize = 4096;
+
+  Index n = vec.size();
+  Index blockEnd = numext::round_down(n, blockSize);
+  for (Index i = 0; i < blockEnd; i += blockSize) {
+    internal::stable_norm_kernel(vec.template segment<blockSize>(i), ssq, scale, invScale);
+  }
+  if (n > blockEnd) {
+    internal::stable_norm_kernel(vec.tail(n - blockEnd), ssq, scale, invScale);
+  }
+}
+
+template <typename VectorType>
+typename VectorType::RealScalar stable_norm_impl(const VectorType& vec,
+                                                 std::enable_if_t<VectorType::IsVectorAtCompileTime>* = 0) {
+  using std::abs;
+  using std::sqrt;
+
+  Index n = vec.size();
+  if (EIGEN_PREDICT_FALSE(n == 1)) return abs(vec.coeff(0));
+
+  typedef typename VectorType::RealScalar RealScalar;
+  RealScalar scale(0);
+  RealScalar invScale(1);
+  RealScalar ssq(0);  // sum of squares
+
+  stable_norm_impl_inner_step(vec, ssq, scale, invScale);
+
+  return scale * sqrt(ssq);
+}
+
+template <typename MatrixType>
+typename MatrixType::RealScalar stable_norm_impl(const MatrixType& mat,
+                                                 std::enable_if_t<!MatrixType::IsVectorAtCompileTime>* = 0) {
+  using std::sqrt;
+
+  typedef typename MatrixType::RealScalar RealScalar;
+  RealScalar scale(0);
+  RealScalar invScale(1);
+  RealScalar ssq(0);  // sum of squares
+
+  for (Index j = 0; j < mat.outerSize(); ++j) stable_norm_impl_inner_step(mat.innerVector(j), ssq, scale, invScale);
+  return scale * sqrt(ssq);
+}
+
+template <typename Derived>
+inline typename NumTraits<typename traits<Derived>::Scalar>::Real blueNorm_impl(const EigenBase<Derived>& _vec) {
+  typedef typename Derived::RealScalar RealScalar;
+  using std::abs;
+  using std::pow;
+  using std::sqrt;
+
+  // This program calculates the machine-dependent constants
+  // bl, b2, slm, s2m, relerr overfl
+  // from the "basic" machine-dependent numbers
+  // nbig, ibeta, it, iemin, iemax, rbig.
+  // The following define the basic machine-dependent constants.
+  // For portability, the PORT subprograms "ilmaeh" and "rlmach"
+  // are used. For any specific computer, each of the assignment
+  // statements can be replaced
+  static const int ibeta = std::numeric_limits<RealScalar>::radix;  // base for floating-point numbers
+  static const int it = NumTraits<RealScalar>::digits();            // number of base-beta digits in mantissa
+  static const int iemin = NumTraits<RealScalar>::min_exponent();   // minimum exponent
+  static const int iemax = NumTraits<RealScalar>::max_exponent();   // maximum exponent
+  static const RealScalar rbig = NumTraits<RealScalar>::highest();  // largest floating-point number
+  static const RealScalar b1 =
+      RealScalar(pow(RealScalar(ibeta), RealScalar(-((1 - iemin) / 2))));  // lower boundary of midrange
+  static const RealScalar b2 =
+      RealScalar(pow(RealScalar(ibeta), RealScalar((iemax + 1 - it) / 2)));  // upper boundary of midrange
+  static const RealScalar s1m =
+      RealScalar(pow(RealScalar(ibeta), RealScalar((2 - iemin) / 2)));  // scaling factor for lower range
+  static const RealScalar s2m =
+      RealScalar(pow(RealScalar(ibeta), RealScalar(-((iemax + it) / 2))));  // scaling factor for upper range
+  static const RealScalar eps = RealScalar(pow(double(ibeta), 1 - it));
+  static const RealScalar relerr = sqrt(eps);  // tolerance for neglecting asml
+
+  const Derived& vec(_vec.derived());
+  Index n = vec.size();
+  RealScalar ab2 = b2 / RealScalar(n);
+  RealScalar asml = RealScalar(0);
+  RealScalar amed = RealScalar(0);
+  RealScalar abig = RealScalar(0);
+
+  for (Index j = 0; j < vec.outerSize(); ++j) {
+    for (typename Derived::InnerIterator iter(vec, j); iter; ++iter) {
+      RealScalar ax = abs(iter.value());
+      if (ax > ab2)
+        abig += numext::abs2(ax * s2m);
+      else if (ax < b1)
+        asml += numext::abs2(ax * s1m);
+      else
+        amed += numext::abs2(ax);
+    }
+  }
+  if (amed != amed) return amed;  // we got a NaN
+  if (abig > RealScalar(0)) {
+    abig = sqrt(abig);
+    if (abig > rbig)  // overflow, or *this contains INF values
+      return abig;    // return INF
+    if (amed > RealScalar(0)) {
+      abig = abig / s2m;
+      amed = sqrt(amed);
+    } else
+      return abig / s2m;
+  } else if (asml > RealScalar(0)) {
+    if (amed > RealScalar(0)) {
+      abig = sqrt(amed);
+      amed = sqrt(asml) / s1m;
+    } else
+      return sqrt(asml) / s1m;
+  } else
+    return sqrt(amed);
+  asml = numext::mini(abig, amed);
+  abig = numext::maxi(abig, amed);
+  if (asml <= abig * relerr)
+    return abig;
+  else
+    return abig * sqrt(RealScalar(1) + numext::abs2(asml / abig));
+}
+
+}  // end namespace internal
+
+/** \returns the \em l2 norm of \c *this avoiding underflow and overflow.
+ * This version use a blockwise two passes algorithm:
+ *  1 - find the absolute largest coefficient \c s
+ *  2 - compute \f$ s \Vert \frac{*this}{s} \Vert \f$ in a standard way
+ *
+ * For architecture/scalar types supporting vectorization, this version
+ * is faster than blueNorm(). Otherwise the blueNorm() is much faster.
+ *
+ * \sa norm(), blueNorm(), hypotNorm()
+ */
+template <typename Derived>
+inline typename NumTraits<typename internal::traits<Derived>::Scalar>::Real MatrixBase<Derived>::stableNorm() const {
+  return internal::stable_norm_impl(derived());
+}
+
+/** \returns the \em l2 norm of \c *this using the Blue's algorithm.
+ * A Portable Fortran Program to Find the Euclidean Norm of a Vector,
+ * ACM TOMS, Vol 4, Issue 1, 1978.
+ *
+ * For architecture/scalar types without vectorization, this version
+ * is much faster than stableNorm(). Otherwise the stableNorm() is faster.
+ *
+ * \sa norm(), stableNorm(), hypotNorm()
+ */
+template <typename Derived>
+inline typename NumTraits<typename internal::traits<Derived>::Scalar>::Real MatrixBase<Derived>::blueNorm() const {
+  return internal::blueNorm_impl(*this);
+}
+
+/** \returns the \em l2 norm of \c *this avoiding underflow and overflow.
+ * This version use a concatenation of hypot() calls, and it is very slow.
+ *
+ * \sa norm(), stableNorm()
+ */
+template <typename Derived>
+inline typename NumTraits<typename internal::traits<Derived>::Scalar>::Real MatrixBase<Derived>::hypotNorm() const {
+  if (size() == 1)
+    return numext::abs(coeff(0, 0));
+  else
+    return this->cwiseAbs().redux(internal::scalar_hypot_op<RealScalar>());
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_STABLENORM_H

include/Eigen/src/Core/StlIterators.h

@@ -0,0 +1,620 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2018 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_STLITERATORS_H
+#define EIGEN_STLITERATORS_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename IteratorType>
+struct indexed_based_stl_iterator_traits;
+
+template <typename Derived>
+class indexed_based_stl_iterator_base {
+ protected:
+  typedef indexed_based_stl_iterator_traits<Derived> traits;
+  typedef typename traits::XprType XprType;
+  typedef indexed_based_stl_iterator_base<typename traits::non_const_iterator> non_const_iterator;
+  typedef indexed_based_stl_iterator_base<typename traits::const_iterator> const_iterator;
+  typedef std::conditional_t<internal::is_const<XprType>::value, non_const_iterator, const_iterator> other_iterator;
+  // NOTE: in C++03 we cannot declare friend classes through typedefs because we need to write friend class:
+  friend class indexed_based_stl_iterator_base<typename traits::const_iterator>;
+  friend class indexed_based_stl_iterator_base<typename traits::non_const_iterator>;
+
+ public:
+  typedef Index difference_type;
+  typedef std::random_access_iterator_tag iterator_category;
+
+  indexed_based_stl_iterator_base() EIGEN_NO_THROW : mp_xpr(0), m_index(0) {}
+  indexed_based_stl_iterator_base(XprType& xpr, Index index) EIGEN_NO_THROW : mp_xpr(&xpr), m_index(index) {}
+
+  indexed_based_stl_iterator_base(const non_const_iterator& other) EIGEN_NO_THROW : mp_xpr(other.mp_xpr),
+                                                                                    m_index(other.m_index) {}
+
+  indexed_based_stl_iterator_base& operator=(const non_const_iterator& other) {
+    mp_xpr = other.mp_xpr;
+    m_index = other.m_index;
+    return *this;
+  }
+
+  Derived& operator++() {
+    ++m_index;
+    return derived();
+  }
+  Derived& operator--() {
+    --m_index;
+    return derived();
+  }
+
+  Derived operator++(int) {
+    Derived prev(derived());
+    operator++();
+    return prev;
+  }
+  Derived operator--(int) {
+    Derived prev(derived());
+    operator--();
+    return prev;
+  }
+
+  friend Derived operator+(const indexed_based_stl_iterator_base& a, Index b) {
+    Derived ret(a.derived());
+    ret += b;
+    return ret;
+  }
+  friend Derived operator-(const indexed_based_stl_iterator_base& a, Index b) {
+    Derived ret(a.derived());
+    ret -= b;
+    return ret;
+  }
+  friend Derived operator+(Index a, const indexed_based_stl_iterator_base& b) {
+    Derived ret(b.derived());
+    ret += a;
+    return ret;
+  }
+  friend Derived operator-(Index a, const indexed_based_stl_iterator_base& b) {
+    Derived ret(b.derived());
+    ret -= a;
+    return ret;
+  }
+
+  Derived& operator+=(Index b) {
+    m_index += b;
+    return derived();
+  }
+  Derived& operator-=(Index b) {
+    m_index -= b;
+    return derived();
+  }
+
+  difference_type operator-(const indexed_based_stl_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index - other.m_index;
+  }
+
+  difference_type operator-(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index - other.m_index;
+  }
+
+  bool operator==(const indexed_based_stl_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index == other.m_index;
+  }
+  bool operator!=(const indexed_based_stl_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index != other.m_index;
+  }
+  bool operator<(const indexed_based_stl_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index < other.m_index;
+  }
+  bool operator<=(const indexed_based_stl_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index <= other.m_index;
+  }
+  bool operator>(const indexed_based_stl_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index > other.m_index;
+  }
+  bool operator>=(const indexed_based_stl_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index >= other.m_index;
+  }
+
+  bool operator==(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index == other.m_index;
+  }
+  bool operator!=(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index != other.m_index;
+  }
+  bool operator<(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index < other.m_index;
+  }
+  bool operator<=(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index <= other.m_index;
+  }
+  bool operator>(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index > other.m_index;
+  }
+  bool operator>=(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index >= other.m_index;
+  }
+
+ protected:
+  Derived& derived() { return static_cast<Derived&>(*this); }
+  const Derived& derived() const { return static_cast<const Derived&>(*this); }
+
+  XprType* mp_xpr;
+  Index m_index;
+};
+
+template <typename Derived>
+class indexed_based_stl_reverse_iterator_base {
+ protected:
+  typedef indexed_based_stl_iterator_traits<Derived> traits;
+  typedef typename traits::XprType XprType;
+  typedef indexed_based_stl_reverse_iterator_base<typename traits::non_const_iterator> non_const_iterator;
+  typedef indexed_based_stl_reverse_iterator_base<typename traits::const_iterator> const_iterator;
+  typedef std::conditional_t<internal::is_const<XprType>::value, non_const_iterator, const_iterator> other_iterator;
+  // NOTE: in C++03 we cannot declare friend classes through typedefs because we need to write friend class:
+  friend class indexed_based_stl_reverse_iterator_base<typename traits::const_iterator>;
+  friend class indexed_based_stl_reverse_iterator_base<typename traits::non_const_iterator>;
+
+ public:
+  typedef Index difference_type;
+  typedef std::random_access_iterator_tag iterator_category;
+
+  indexed_based_stl_reverse_iterator_base() : mp_xpr(0), m_index(0) {}
+  indexed_based_stl_reverse_iterator_base(XprType& xpr, Index index) : mp_xpr(&xpr), m_index(index) {}
+
+  indexed_based_stl_reverse_iterator_base(const non_const_iterator& other)
+      : mp_xpr(other.mp_xpr), m_index(other.m_index) {}
+
+  indexed_based_stl_reverse_iterator_base& operator=(const non_const_iterator& other) {
+    mp_xpr = other.mp_xpr;
+    m_index = other.m_index;
+    return *this;
+  }
+
+  Derived& operator++() {
+    --m_index;
+    return derived();
+  }
+  Derived& operator--() {
+    ++m_index;
+    return derived();
+  }
+
+  Derived operator++(int) {
+    Derived prev(derived());
+    operator++();
+    return prev;
+  }
+  Derived operator--(int) {
+    Derived prev(derived());
+    operator--();
+    return prev;
+  }
+
+  friend Derived operator+(const indexed_based_stl_reverse_iterator_base& a, Index b) {
+    Derived ret(a.derived());
+    ret += b;
+    return ret;
+  }
+  friend Derived operator-(const indexed_based_stl_reverse_iterator_base& a, Index b) {
+    Derived ret(a.derived());
+    ret -= b;
+    return ret;
+  }
+  friend Derived operator+(Index a, const indexed_based_stl_reverse_iterator_base& b) {
+    Derived ret(b.derived());
+    ret += a;
+    return ret;
+  }
+  friend Derived operator-(Index a, const indexed_based_stl_reverse_iterator_base& b) {
+    Derived ret(b.derived());
+    ret -= a;
+    return ret;
+  }
+
+  Derived& operator+=(Index b) {
+    m_index -= b;
+    return derived();
+  }
+  Derived& operator-=(Index b) {
+    m_index += b;
+    return derived();
+  }
+
+  difference_type operator-(const indexed_based_stl_reverse_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return other.m_index - m_index;
+  }
+
+  difference_type operator-(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return other.m_index - m_index;
+  }
+
+  bool operator==(const indexed_based_stl_reverse_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index == other.m_index;
+  }
+  bool operator!=(const indexed_based_stl_reverse_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index != other.m_index;
+  }
+  bool operator<(const indexed_based_stl_reverse_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index > other.m_index;
+  }
+  bool operator<=(const indexed_based_stl_reverse_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index >= other.m_index;
+  }
+  bool operator>(const indexed_based_stl_reverse_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index < other.m_index;
+  }
+  bool operator>=(const indexed_based_stl_reverse_iterator_base& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index <= other.m_index;
+  }
+
+  bool operator==(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index == other.m_index;
+  }
+  bool operator!=(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index != other.m_index;
+  }
+  bool operator<(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index > other.m_index;
+  }
+  bool operator<=(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index >= other.m_index;
+  }
+  bool operator>(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index < other.m_index;
+  }
+  bool operator>=(const other_iterator& other) const {
+    eigen_assert(mp_xpr == other.mp_xpr);
+    return m_index <= other.m_index;
+  }
+
+ protected:
+  Derived& derived() { return static_cast<Derived&>(*this); }
+  const Derived& derived() const { return static_cast<const Derived&>(*this); }
+
+  XprType* mp_xpr;
+  Index m_index;
+};
+
+template <typename XprType>
+class pointer_based_stl_iterator {
+  enum { is_lvalue = internal::is_lvalue<XprType>::value };
+  typedef pointer_based_stl_iterator<std::remove_const_t<XprType>> non_const_iterator;
+  typedef pointer_based_stl_iterator<std::add_const_t<XprType>> const_iterator;
+  typedef std::conditional_t<internal::is_const<XprType>::value, non_const_iterator, const_iterator> other_iterator;
+  // NOTE: in C++03 we cannot declare friend classes through typedefs because we need to write friend class:
+  friend class pointer_based_stl_iterator<std::add_const_t<XprType>>;
+  friend class pointer_based_stl_iterator<std::remove_const_t<XprType>>;
+
+ public:
+  typedef Index difference_type;
+  typedef typename XprType::Scalar value_type;
+#if EIGEN_CPLUSPLUS >= 202002L
+  typedef std::conditional_t<XprType::InnerStrideAtCompileTime == 1, std::contiguous_iterator_tag,
+                             std::random_access_iterator_tag>
+      iterator_category;
+#else
+  typedef std::random_access_iterator_tag iterator_category;
+#endif
+  typedef std::conditional_t<bool(is_lvalue), value_type*, const value_type*> pointer;
+  typedef std::conditional_t<bool(is_lvalue), value_type&, const value_type&> reference;
+
+  pointer_based_stl_iterator() EIGEN_NO_THROW : m_ptr(0) {}
+  pointer_based_stl_iterator(XprType& xpr, Index index) EIGEN_NO_THROW : m_incr(xpr.innerStride()) {
+    m_ptr = xpr.data() + index * m_incr.value();
+  }
+
+  pointer_based_stl_iterator(const non_const_iterator& other) EIGEN_NO_THROW : m_ptr(other.m_ptr),
+                                                                               m_incr(other.m_incr) {}
+
+  pointer_based_stl_iterator& operator=(const non_const_iterator& other) EIGEN_NO_THROW {
+    m_ptr = other.m_ptr;
+    m_incr.setValue(other.m_incr);
+    return *this;
+  }
+
+  reference operator*() const { return *m_ptr; }
+  reference operator[](Index i) const { return *(m_ptr + i * m_incr.value()); }
+  pointer operator->() const { return m_ptr; }
+
+  pointer_based_stl_iterator& operator++() {
+    m_ptr += m_incr.value();
+    return *this;
+  }
+  pointer_based_stl_iterator& operator--() {
+    m_ptr -= m_incr.value();
+    return *this;
+  }
+
+  pointer_based_stl_iterator operator++(int) {
+    pointer_based_stl_iterator prev(*this);
+    operator++();
+    return prev;
+  }
+  pointer_based_stl_iterator operator--(int) {
+    pointer_based_stl_iterator prev(*this);
+    operator--();
+    return prev;
+  }
+
+  friend pointer_based_stl_iterator operator+(const pointer_based_stl_iterator& a, Index b) {
+    pointer_based_stl_iterator ret(a);
+    ret += b;
+    return ret;
+  }
+  friend pointer_based_stl_iterator operator-(const pointer_based_stl_iterator& a, Index b) {
+    pointer_based_stl_iterator ret(a);
+    ret -= b;
+    return ret;
+  }
+  friend pointer_based_stl_iterator operator+(Index a, const pointer_based_stl_iterator& b) {
+    pointer_based_stl_iterator ret(b);
+    ret += a;
+    return ret;
+  }
+  friend pointer_based_stl_iterator operator-(Index a, const pointer_based_stl_iterator& b) {
+    pointer_based_stl_iterator ret(b);
+    ret -= a;
+    return ret;
+  }
+
+  pointer_based_stl_iterator& operator+=(Index b) {
+    m_ptr += b * m_incr.value();
+    return *this;
+  }
+  pointer_based_stl_iterator& operator-=(Index b) {
+    m_ptr -= b * m_incr.value();
+    return *this;
+  }
+
+  difference_type operator-(const pointer_based_stl_iterator& other) const {
+    return (m_ptr - other.m_ptr) / m_incr.value();
+  }
+
+  difference_type operator-(const other_iterator& other) const { return (m_ptr - other.m_ptr) / m_incr.value(); }
+
+  bool operator==(const pointer_based_stl_iterator& other) const { return m_ptr == other.m_ptr; }
+  bool operator!=(const pointer_based_stl_iterator& other) const { return m_ptr != other.m_ptr; }
+  bool operator<(const pointer_based_stl_iterator& other) const { return m_ptr < other.m_ptr; }
+  bool operator<=(const pointer_based_stl_iterator& other) const { return m_ptr <= other.m_ptr; }
+  bool operator>(const pointer_based_stl_iterator& other) const { return m_ptr > other.m_ptr; }
+  bool operator>=(const pointer_based_stl_iterator& other) const { return m_ptr >= other.m_ptr; }
+
+  bool operator==(const other_iterator& other) const { return m_ptr == other.m_ptr; }
+  bool operator!=(const other_iterator& other) const { return m_ptr != other.m_ptr; }
+  bool operator<(const other_iterator& other) const { return m_ptr < other.m_ptr; }
+  bool operator<=(const other_iterator& other) const { return m_ptr <= other.m_ptr; }
+  bool operator>(const other_iterator& other) const { return m_ptr > other.m_ptr; }
+  bool operator>=(const other_iterator& other) const { return m_ptr >= other.m_ptr; }
+
+ protected:
+  pointer m_ptr;
+  internal::variable_if_dynamic<Index, XprType::InnerStrideAtCompileTime> m_incr;
+};
+
+template <typename XprType_>
+struct indexed_based_stl_iterator_traits<generic_randaccess_stl_iterator<XprType_>> {
+  typedef XprType_ XprType;
+  typedef generic_randaccess_stl_iterator<std::remove_const_t<XprType>> non_const_iterator;
+  typedef generic_randaccess_stl_iterator<std::add_const_t<XprType>> const_iterator;
+};
+
+template <typename XprType>
+class generic_randaccess_stl_iterator
+    : public indexed_based_stl_iterator_base<generic_randaccess_stl_iterator<XprType>> {
+ public:
+  typedef typename XprType::Scalar value_type;
+
+ protected:
+  enum {
+    has_direct_access = (internal::traits<XprType>::Flags & DirectAccessBit) ? 1 : 0,
+    is_lvalue = internal::is_lvalue<XprType>::value
+  };
+
+  typedef indexed_based_stl_iterator_base<generic_randaccess_stl_iterator> Base;
+  using Base::m_index;
+  using Base::mp_xpr;
+
+  // TODO currently const Transpose/Reshape expressions never returns const references,
+  // so lets return by value too.
+  // typedef std::conditional_t<bool(has_direct_access), const value_type&, const value_type> read_only_ref_t;
+  typedef const value_type read_only_ref_t;
+
+ public:
+  typedef std::conditional_t<bool(is_lvalue), value_type*, const value_type*> pointer;
+  typedef std::conditional_t<bool(is_lvalue), value_type&, read_only_ref_t> reference;
+
+  generic_randaccess_stl_iterator() : Base() {}
+  generic_randaccess_stl_iterator(XprType& xpr, Index index) : Base(xpr, index) {}
+  generic_randaccess_stl_iterator(const typename Base::non_const_iterator& other) : Base(other) {}
+  using Base::operator=;
+
+  reference operator*() const { return (*mp_xpr)(m_index); }
+  reference operator[](Index i) const { return (*mp_xpr)(m_index + i); }
+  pointer operator->() const { return &((*mp_xpr)(m_index)); }
+};
+
+template <typename XprType_, DirectionType Direction>
+struct indexed_based_stl_iterator_traits<subvector_stl_iterator<XprType_, Direction>> {
+  typedef XprType_ XprType;
+  typedef subvector_stl_iterator<std::remove_const_t<XprType>, Direction> non_const_iterator;
+  typedef subvector_stl_iterator<std::add_const_t<XprType>, Direction> const_iterator;
+};
+
+template <typename XprType, DirectionType Direction>
+class subvector_stl_iterator : public indexed_based_stl_iterator_base<subvector_stl_iterator<XprType, Direction>> {
+ protected:
+  enum { is_lvalue = internal::is_lvalue<XprType>::value };
+
+  typedef indexed_based_stl_iterator_base<subvector_stl_iterator> Base;
+  using Base::m_index;
+  using Base::mp_xpr;
+
+  typedef std::conditional_t<Direction == Vertical, typename XprType::ColXpr, typename XprType::RowXpr> SubVectorType;
+  typedef std::conditional_t<Direction == Vertical, typename XprType::ConstColXpr, typename XprType::ConstRowXpr>
+      ConstSubVectorType;
+
+ public:
+  typedef std::conditional_t<bool(is_lvalue), SubVectorType, ConstSubVectorType> reference;
+  typedef typename reference::PlainObject value_type;
+
+ private:
+  class subvector_stl_iterator_ptr {
+   public:
+    subvector_stl_iterator_ptr(const reference& subvector) : m_subvector(subvector) {}
+    reference* operator->() { return &m_subvector; }
+
+   private:
+    reference m_subvector;
+  };
+
+ public:
+  typedef subvector_stl_iterator_ptr pointer;
+
+  subvector_stl_iterator() : Base() {}
+  subvector_stl_iterator(XprType& xpr, Index index) : Base(xpr, index) {}
+
+  reference operator*() const { return (*mp_xpr).template subVector<Direction>(m_index); }
+  reference operator[](Index i) const { return (*mp_xpr).template subVector<Direction>(m_index + i); }
+  pointer operator->() const { return (*mp_xpr).template subVector<Direction>(m_index); }
+};
+
+template <typename XprType_, DirectionType Direction>
+struct indexed_based_stl_iterator_traits<subvector_stl_reverse_iterator<XprType_, Direction>> {
+  typedef XprType_ XprType;
+  typedef subvector_stl_reverse_iterator<std::remove_const_t<XprType>, Direction> non_const_iterator;
+  typedef subvector_stl_reverse_iterator<std::add_const_t<XprType>, Direction> const_iterator;
+};
+
+template <typename XprType, DirectionType Direction>
+class subvector_stl_reverse_iterator
+    : public indexed_based_stl_reverse_iterator_base<subvector_stl_reverse_iterator<XprType, Direction>> {
+ protected:
+  enum { is_lvalue = internal::is_lvalue<XprType>::value };
+
+  typedef indexed_based_stl_reverse_iterator_base<subvector_stl_reverse_iterator> Base;
+  using Base::m_index;
+  using Base::mp_xpr;
+
+  typedef std::conditional_t<Direction == Vertical, typename XprType::ColXpr, typename XprType::RowXpr> SubVectorType;
+  typedef std::conditional_t<Direction == Vertical, typename XprType::ConstColXpr, typename XprType::ConstRowXpr>
+      ConstSubVectorType;
+
+ public:
+  typedef std::conditional_t<bool(is_lvalue), SubVectorType, ConstSubVectorType> reference;
+  typedef typename reference::PlainObject value_type;
+
+ private:
+  class subvector_stl_reverse_iterator_ptr {
+   public:
+    subvector_stl_reverse_iterator_ptr(const reference& subvector) : m_subvector(subvector) {}
+    reference* operator->() { return &m_subvector; }
+
+   private:
+    reference m_subvector;
+  };
+
+ public:
+  typedef subvector_stl_reverse_iterator_ptr pointer;
+
+  subvector_stl_reverse_iterator() : Base() {}
+  subvector_stl_reverse_iterator(XprType& xpr, Index index) : Base(xpr, index) {}
+
+  reference operator*() const { return (*mp_xpr).template subVector<Direction>(m_index); }
+  reference operator[](Index i) const { return (*mp_xpr).template subVector<Direction>(m_index + i); }
+  pointer operator->() const { return (*mp_xpr).template subVector<Direction>(m_index); }
+};
+
+}  // namespace internal
+
+/** returns an iterator to the first element of the 1D vector or array
+ * \only_for_vectors
+ * \sa end(), cbegin()
+ */
+template <typename Derived>
+inline typename DenseBase<Derived>::iterator DenseBase<Derived>::begin() {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived);
+  return iterator(derived(), 0);
+}
+
+/** const version of begin() */
+template <typename Derived>
+inline typename DenseBase<Derived>::const_iterator DenseBase<Derived>::begin() const {
+  return cbegin();
+}
+
+/** returns a read-only const_iterator to the first element of the 1D vector or array
+ * \only_for_vectors
+ * \sa cend(), begin()
+ */
+template <typename Derived>
+inline typename DenseBase<Derived>::const_iterator DenseBase<Derived>::cbegin() const {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived);
+  return const_iterator(derived(), 0);
+}
+
+/** returns an iterator to the element following the last element of the 1D vector or array
+ * \only_for_vectors
+ * \sa begin(), cend()
+ */
+template <typename Derived>
+inline typename DenseBase<Derived>::iterator DenseBase<Derived>::end() {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived);
+  return iterator(derived(), size());
+}
+
+/** const version of end() */
+template <typename Derived>
+inline typename DenseBase<Derived>::const_iterator DenseBase<Derived>::end() const {
+  return cend();
+}
+
+/** returns a read-only const_iterator to the element following the last element of the 1D vector or array
+ * \only_for_vectors
+ * \sa begin(), cend()
+ */
+template <typename Derived>
+inline typename DenseBase<Derived>::const_iterator DenseBase<Derived>::cend() const {
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived);
+  return const_iterator(derived(), size());
+}
+
+}  // namespace Eigen
+
+#endif  // EIGEN_STLITERATORS_H

include/Eigen/src/Core/Stride.h

@@ -0,0 +1,114 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2010 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_STRIDE_H
+#define EIGEN_STRIDE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class Stride
+ * \ingroup Core_Module
+ *
+ * \brief Holds strides information for Map
+ *
+ * This class holds the strides information for mapping arrays with strides with class Map.
+ *
+ * It holds two values: the inner stride and the outer stride.
+ *
+ * The inner stride is the pointer increment between two consecutive entries within a given row of a
+ * row-major matrix or within a given column of a column-major matrix.
+ *
+ * The outer stride is the pointer increment between two consecutive rows of a row-major matrix or
+ * between two consecutive columns of a column-major matrix.
+ *
+ * These two values can be passed either at compile-time as template parameters, or at runtime as
+ * arguments to the constructor.
+ *
+ * Indeed, this class takes two template parameters:
+ *  \tparam OuterStrideAtCompileTime_ the outer stride, or Dynamic if you want to specify it at runtime.
+ *  \tparam InnerStrideAtCompileTime_ the inner stride, or Dynamic if you want to specify it at runtime.
+ *
+ * Here is an example:
+ * \include Map_general_stride.cpp
+ * Output: \verbinclude Map_general_stride.out
+ *
+ * Both strides can be negative. However, a negative stride of -1 cannot be specified at compile time
+ * because of the ambiguity with Dynamic which is defined to -1 (historically, negative strides were
+ * not allowed).
+ *
+ * Note that for compile-time vectors (ColsAtCompileTime==1 or RowsAtCompile==1),
+ * the inner stride is the pointer increment between two consecutive elements,
+ * regardless of storage layout.
+ *
+ * \sa class InnerStride, class OuterStride, \ref TopicStorageOrders
+ */
+template <int OuterStrideAtCompileTime_, int InnerStrideAtCompileTime_>
+class Stride {
+ public:
+  typedef Eigen::Index Index;  ///< \deprecated since Eigen 3.3
+  enum { InnerStrideAtCompileTime = InnerStrideAtCompileTime_, OuterStrideAtCompileTime = OuterStrideAtCompileTime_ };
+
+  /** Default constructor, for use when strides are fixed at compile time */
+  EIGEN_DEVICE_FUNC Stride() : m_outer(OuterStrideAtCompileTime), m_inner(InnerStrideAtCompileTime) {
+    // FIXME: for Eigen 4 we should use DynamicIndex instead of Dynamic.
+    // FIXME: for Eigen 4 we should also unify this API with fix<>
+    eigen_assert(InnerStrideAtCompileTime != Dynamic && OuterStrideAtCompileTime != Dynamic);
+  }
+
+  /** Constructor allowing to pass the strides at runtime */
+  EIGEN_DEVICE_FUNC Stride(Index outerStride, Index innerStride) : m_outer(outerStride), m_inner(innerStride) {}
+
+  /** Copy constructor */
+  EIGEN_DEVICE_FUNC Stride(const Stride& other) : m_outer(other.outer()), m_inner(other.inner()) {}
+
+  /** Copy assignment operator */
+  EIGEN_DEVICE_FUNC Stride& operator=(const Stride& other) {
+    m_outer.setValue(other.outer());
+    m_inner.setValue(other.inner());
+    return *this;
+  }
+
+  /** \returns the outer stride */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outer() const { return m_outer.value(); }
+  /** \returns the inner stride */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index inner() const { return m_inner.value(); }
+
+ protected:
+  internal::variable_if_dynamic<Index, OuterStrideAtCompileTime> m_outer;
+  internal::variable_if_dynamic<Index, InnerStrideAtCompileTime> m_inner;
+};
+
+/** \brief Convenience specialization of Stride to specify only an inner stride
+ * See class Map for some examples */
+template <int Value>
+class InnerStride : public Stride<0, Value> {
+  typedef Stride<0, Value> Base;
+
+ public:
+  EIGEN_DEVICE_FUNC InnerStride() : Base() {}
+  EIGEN_DEVICE_FUNC InnerStride(Index v) : Base(0, v) {}  // FIXME making this explicit could break valid code
+};
+
+/** \brief Convenience specialization of Stride to specify only an outer stride
+ * See class Map for some examples */
+template <int Value>
+class OuterStride : public Stride<Value, 0> {
+  typedef Stride<Value, 0> Base;
+
+ public:
+  EIGEN_DEVICE_FUNC OuterStride() : Base() {}
+  EIGEN_DEVICE_FUNC OuterStride(Index v) : Base(v, 0) {}  // FIXME making this explicit could break valid code
+};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_STRIDE_H

include/Eigen/src/Core/Swap.h

@@ -0,0 +1,74 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_SWAP_H
+#define EIGEN_SWAP_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+// Overload default assignPacket behavior for swapping them
+template <typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT>
+class generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT,
+                                      swap_assign_op<typename DstEvaluatorTypeT::Scalar>, Specialized>
+    : public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT,
+                                             swap_assign_op<typename DstEvaluatorTypeT::Scalar>, BuiltIn> {
+ protected:
+  typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT,
+                                          swap_assign_op<typename DstEvaluatorTypeT::Scalar>, BuiltIn>
+      Base;
+  using Base::m_dst;
+  using Base::m_functor;
+  using Base::m_src;
+
+ public:
+  typedef typename Base::Scalar Scalar;
+  typedef typename Base::DstXprType DstXprType;
+  typedef swap_assign_op<Scalar> Functor;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE generic_dense_assignment_kernel(DstEvaluatorTypeT &dst,
+                                                                        const SrcEvaluatorTypeT &src,
+                                                                        const Functor &func, DstXprType &dstExpr)
+      : Base(dst, src, func, dstExpr) {}
+
+  template <int StoreMode, int LoadMode, typename PacketType>
+  EIGEN_STRONG_INLINE void assignPacket(Index row, Index col) {
+    PacketType tmp = m_src.template packet<LoadMode, PacketType>(row, col);
+    const_cast<SrcEvaluatorTypeT &>(m_src).template writePacket<LoadMode>(
+        row, col, m_dst.template packet<StoreMode, PacketType>(row, col));
+    m_dst.template writePacket<StoreMode>(row, col, tmp);
+  }
+
+  template <int StoreMode, int LoadMode, typename PacketType>
+  EIGEN_STRONG_INLINE void assignPacket(Index index) {
+    PacketType tmp = m_src.template packet<LoadMode, PacketType>(index);
+    const_cast<SrcEvaluatorTypeT &>(m_src).template writePacket<LoadMode>(
+        index, m_dst.template packet<StoreMode, PacketType>(index));
+    m_dst.template writePacket<StoreMode>(index, tmp);
+  }
+
+  // TODO find a simple way not to have to copy/paste this function from generic_dense_assignment_kernel, by simple I
+  // mean no CRTP (Gael)
+  template <int StoreMode, int LoadMode, typename PacketType>
+  EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner) {
+    Index row = Base::rowIndexByOuterInner(outer, inner);
+    Index col = Base::colIndexByOuterInner(outer, inner);
+    assignPacket<StoreMode, LoadMode, PacketType>(row, col);
+  }
+};
+
+}  // namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_SWAP_H

include/Eigen/src/Core/Transpose.h

@@ -0,0 +1,427 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2009-2014 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_TRANSPOSE_H
+#define EIGEN_TRANSPOSE_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename MatrixType>
+struct traits<Transpose<MatrixType> > : public traits<MatrixType> {
+  typedef typename ref_selector<MatrixType>::type MatrixTypeNested;
+  typedef std::remove_reference_t<MatrixTypeNested> MatrixTypeNestedPlain;
+  enum {
+    RowsAtCompileTime = MatrixType::ColsAtCompileTime,
+    ColsAtCompileTime = MatrixType::RowsAtCompileTime,
+    MaxRowsAtCompileTime = MatrixType::MaxColsAtCompileTime,
+    MaxColsAtCompileTime = MatrixType::MaxRowsAtCompileTime,
+    FlagsLvalueBit = is_lvalue<MatrixType>::value ? LvalueBit : 0,
+    Flags0 = traits<MatrixTypeNestedPlain>::Flags & ~(LvalueBit | NestByRefBit),
+    Flags1 = Flags0 | FlagsLvalueBit,
+    Flags = Flags1 ^ RowMajorBit,
+    InnerStrideAtCompileTime = inner_stride_at_compile_time<MatrixType>::ret,
+    OuterStrideAtCompileTime = outer_stride_at_compile_time<MatrixType>::ret
+  };
+};
+}  // namespace internal
+
+template <typename MatrixType, typename StorageKind>
+class TransposeImpl;
+
+/** \class Transpose
+ * \ingroup Core_Module
+ *
+ * \brief Expression of the transpose of a matrix
+ *
+ * \tparam MatrixType the type of the object of which we are taking the transpose
+ *
+ * This class represents an expression of the transpose of a matrix.
+ * It is the return type of MatrixBase::transpose() and MatrixBase::adjoint()
+ * and most of the time this is the only way it is used.
+ *
+ * \sa MatrixBase::transpose(), MatrixBase::adjoint()
+ */
+template <typename MatrixType>
+class Transpose : public TransposeImpl<MatrixType, typename internal::traits<MatrixType>::StorageKind> {
+ public:
+  typedef typename internal::ref_selector<MatrixType>::non_const_type MatrixTypeNested;
+
+  typedef typename TransposeImpl<MatrixType, typename internal::traits<MatrixType>::StorageKind>::Base Base;
+  EIGEN_GENERIC_PUBLIC_INTERFACE(Transpose)
+  typedef internal::remove_all_t<MatrixType> NestedExpression;
+
+  EIGEN_DEVICE_FUNC explicit EIGEN_STRONG_INLINE Transpose(MatrixType& matrix) : m_matrix(matrix) {}
+
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Transpose)
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_matrix.cols(); }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_matrix.rows(); }
+
+  /** \returns the nested expression */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const internal::remove_all_t<MatrixTypeNested>& nestedExpression() const {
+    return m_matrix;
+  }
+
+  /** \returns the nested expression */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::remove_reference_t<MatrixTypeNested>& nestedExpression() {
+    return m_matrix;
+  }
+
+  /** \internal */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resize(Index nrows, Index ncols) { m_matrix.resize(ncols, nrows); }
+
+ protected:
+  typename internal::ref_selector<MatrixType>::non_const_type m_matrix;
+};
+
+namespace internal {
+
+template <typename MatrixType, bool HasDirectAccess = has_direct_access<MatrixType>::ret>
+struct TransposeImpl_base {
+  typedef typename dense_xpr_base<Transpose<MatrixType> >::type type;
+};
+
+template <typename MatrixType>
+struct TransposeImpl_base<MatrixType, false> {
+  typedef typename dense_xpr_base<Transpose<MatrixType> >::type type;
+};
+
+}  // end namespace internal
+
+// Generic API dispatcher
+template <typename XprType, typename StorageKind>
+class TransposeImpl : public internal::generic_xpr_base<Transpose<XprType> >::type {
+ public:
+  typedef typename internal::generic_xpr_base<Transpose<XprType> >::type Base;
+};
+
+template <typename MatrixType>
+class TransposeImpl<MatrixType, Dense> : public internal::TransposeImpl_base<MatrixType>::type {
+ public:
+  typedef typename internal::TransposeImpl_base<MatrixType>::type Base;
+  using Base::coeffRef;
+  EIGEN_DENSE_PUBLIC_INTERFACE(Transpose<MatrixType>)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(TransposeImpl)
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index innerStride() const { return derived().nestedExpression().innerStride(); }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index outerStride() const { return derived().nestedExpression().outerStride(); }
+
+  typedef std::conditional_t<internal::is_lvalue<MatrixType>::value, Scalar, const Scalar> ScalarWithConstIfNotLvalue;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr ScalarWithConstIfNotLvalue* data() {
+    return derived().nestedExpression().data();
+  }
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const Scalar* data() const {
+    return derived().nestedExpression().data();
+  }
+
+  // FIXME: shall we keep the const version of coeffRef?
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeffRef(Index rowId, Index colId) const {
+    return derived().nestedExpression().coeffRef(colId, rowId);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeffRef(Index index) const {
+    return derived().nestedExpression().coeffRef(index);
+  }
+
+ protected:
+  EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(TransposeImpl)
+};
+
+/** \returns an expression of the transpose of *this.
+ *
+ * Example: \include MatrixBase_transpose.cpp
+ * Output: \verbinclude MatrixBase_transpose.out
+ *
+ * \warning If you want to replace a matrix by its own transpose, do \b NOT do this:
+ * \code
+ * m = m.transpose(); // bug!!! caused by aliasing effect
+ * \endcode
+ * Instead, use the transposeInPlace() method:
+ * \code
+ * m.transposeInPlace();
+ * \endcode
+ * which gives Eigen good opportunities for optimization, or alternatively you can also do:
+ * \code
+ * m = m.transpose().eval();
+ * \endcode
+ *
+ * \sa transposeInPlace(), adjoint() */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename DenseBase<Derived>::TransposeReturnType DenseBase<Derived>::transpose() {
+  return TransposeReturnType(derived());
+}
+
+/** This is the const version of transpose().
+ *
+ * Make sure you read the warning for transpose() !
+ *
+ * \sa transposeInPlace(), adjoint() */
+template <typename Derived>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename DenseBase<Derived>::ConstTransposeReturnType
+DenseBase<Derived>::transpose() const {
+  return ConstTransposeReturnType(derived());
+}
+
+/** \returns an expression of the adjoint (i.e. conjugate transpose) of *this.
+ *
+ * Example: \include MatrixBase_adjoint.cpp
+ * Output: \verbinclude MatrixBase_adjoint.out
+ *
+ * \warning If you want to replace a matrix by its own adjoint, do \b NOT do this:
+ * \code
+ * m = m.adjoint(); // bug!!! caused by aliasing effect
+ * \endcode
+ * Instead, use the adjointInPlace() method:
+ * \code
+ * m.adjointInPlace();
+ * \endcode
+ * which gives Eigen good opportunities for optimization, or alternatively you can also do:
+ * \code
+ * m = m.adjoint().eval();
+ * \endcode
+ *
+ * \sa adjointInPlace(), transpose(), conjugate(), class Transpose, class internal::scalar_conjugate_op */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline const typename MatrixBase<Derived>::AdjointReturnType MatrixBase<Derived>::adjoint() const {
+  return AdjointReturnType(this->transpose());
+}
+
+/***************************************************************************
+ * "in place" transpose implementation
+ ***************************************************************************/
+
+namespace internal {
+
+template <typename MatrixType,
+          bool IsSquare = (MatrixType::RowsAtCompileTime == MatrixType::ColsAtCompileTime) &&
+                          MatrixType::RowsAtCompileTime != Dynamic,
+          bool MatchPacketSize =
+              (int(MatrixType::RowsAtCompileTime) == int(internal::packet_traits<typename MatrixType::Scalar>::size)) &&
+              (internal::evaluator<MatrixType>::Flags & PacketAccessBit)>
+struct inplace_transpose_selector;
+
+template <typename MatrixType>
+struct inplace_transpose_selector<MatrixType, true, false> {  // square matrix
+  static void run(MatrixType& m) {
+    m.matrix().template triangularView<StrictlyUpper>().swap(
+        m.matrix().transpose().template triangularView<StrictlyUpper>());
+  }
+};
+
+template <typename MatrixType>
+struct inplace_transpose_selector<MatrixType, true, true> {  // PacketSize x PacketSize
+  static void run(MatrixType& m) {
+    typedef typename MatrixType::Scalar Scalar;
+    typedef typename internal::packet_traits<typename MatrixType::Scalar>::type Packet;
+    const Index PacketSize = internal::packet_traits<Scalar>::size;
+    const Index Alignment = internal::evaluator<MatrixType>::Alignment;
+    PacketBlock<Packet> A;
+    for (Index i = 0; i < PacketSize; ++i) A.packet[i] = m.template packetByOuterInner<Alignment>(i, 0);
+    internal::ptranspose(A);
+    for (Index i = 0; i < PacketSize; ++i)
+      m.template writePacket<Alignment>(m.rowIndexByOuterInner(i, 0), m.colIndexByOuterInner(i, 0), A.packet[i]);
+  }
+};
+
+template <typename MatrixType, Index Alignment>
+void BlockedInPlaceTranspose(MatrixType& m) {
+  typedef typename MatrixType::Scalar Scalar;
+  typedef typename internal::packet_traits<typename MatrixType::Scalar>::type Packet;
+  const Index PacketSize = internal::packet_traits<Scalar>::size;
+  eigen_assert(m.rows() == m.cols());
+  int row_start = 0;
+  for (; row_start + PacketSize <= m.rows(); row_start += PacketSize) {
+    for (int col_start = row_start; col_start + PacketSize <= m.cols(); col_start += PacketSize) {
+      PacketBlock<Packet> A;
+      if (row_start == col_start) {
+        for (Index i = 0; i < PacketSize; ++i)
+          A.packet[i] = m.template packetByOuterInner<Alignment>(row_start + i, col_start);
+        internal::ptranspose(A);
+        for (Index i = 0; i < PacketSize; ++i)
+          m.template writePacket<Alignment>(m.rowIndexByOuterInner(row_start + i, col_start),
+                                            m.colIndexByOuterInner(row_start + i, col_start), A.packet[i]);
+      } else {
+        PacketBlock<Packet> B;
+        for (Index i = 0; i < PacketSize; ++i) {
+          A.packet[i] = m.template packetByOuterInner<Alignment>(row_start + i, col_start);
+          B.packet[i] = m.template packetByOuterInner<Alignment>(col_start + i, row_start);
+        }
+        internal::ptranspose(A);
+        internal::ptranspose(B);
+        for (Index i = 0; i < PacketSize; ++i) {
+          m.template writePacket<Alignment>(m.rowIndexByOuterInner(row_start + i, col_start),
+                                            m.colIndexByOuterInner(row_start + i, col_start), B.packet[i]);
+          m.template writePacket<Alignment>(m.rowIndexByOuterInner(col_start + i, row_start),
+                                            m.colIndexByOuterInner(col_start + i, row_start), A.packet[i]);
+        }
+      }
+    }
+  }
+  for (Index row = row_start; row < m.rows(); ++row) {
+    m.matrix().row(row).head(row).swap(m.matrix().col(row).head(row).transpose());
+  }
+}
+
+template <typename MatrixType, bool MatchPacketSize>
+struct inplace_transpose_selector<MatrixType, false, MatchPacketSize> {  // non square or dynamic matrix
+  static void run(MatrixType& m) {
+    typedef typename MatrixType::Scalar Scalar;
+    if (m.rows() == m.cols()) {
+      const Index PacketSize = internal::packet_traits<Scalar>::size;
+      if (!NumTraits<Scalar>::IsComplex && m.rows() >= PacketSize) {
+        if ((m.rows() % PacketSize) == 0)
+          BlockedInPlaceTranspose<MatrixType, internal::evaluator<MatrixType>::Alignment>(m);
+        else
+          BlockedInPlaceTranspose<MatrixType, Unaligned>(m);
+      } else {
+        m.matrix().template triangularView<StrictlyUpper>().swap(
+            m.matrix().transpose().template triangularView<StrictlyUpper>());
+      }
+    } else {
+      m = m.transpose().eval();
+    }
+  }
+};
+
+}  // end namespace internal
+
+/** This is the "in place" version of transpose(): it replaces \c *this by its own transpose.
+ * Thus, doing
+ * \code
+ * m.transposeInPlace();
+ * \endcode
+ * has the same effect on m as doing
+ * \code
+ * m = m.transpose().eval();
+ * \endcode
+ * and is faster and also safer because in the latter line of code, forgetting the eval() results
+ * in a bug caused by \ref TopicAliasing "aliasing".
+ *
+ * Notice however that this method is only useful if you want to replace a matrix by its own transpose.
+ * If you just need the transpose of a matrix, use transpose().
+ *
+ * \note if the matrix is not square, then \c *this must be a resizable matrix.
+ * This excludes (non-square) fixed-size matrices, block-expressions and maps.
+ *
+ * \sa transpose(), adjoint(), adjointInPlace() */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline void DenseBase<Derived>::transposeInPlace() {
+  eigen_assert((rows() == cols() || (RowsAtCompileTime == Dynamic && ColsAtCompileTime == Dynamic)) &&
+               "transposeInPlace() called on a non-square non-resizable matrix");
+  internal::inplace_transpose_selector<Derived>::run(derived());
+}
+
+/***************************************************************************
+ * "in place" adjoint implementation
+ ***************************************************************************/
+
+/** This is the "in place" version of adjoint(): it replaces \c *this by its own transpose.
+ * Thus, doing
+ * \code
+ * m.adjointInPlace();
+ * \endcode
+ * has the same effect on m as doing
+ * \code
+ * m = m.adjoint().eval();
+ * \endcode
+ * and is faster and also safer because in the latter line of code, forgetting the eval() results
+ * in a bug caused by aliasing.
+ *
+ * Notice however that this method is only useful if you want to replace a matrix by its own adjoint.
+ * If you just need the adjoint of a matrix, use adjoint().
+ *
+ * \note if the matrix is not square, then \c *this must be a resizable matrix.
+ * This excludes (non-square) fixed-size matrices, block-expressions and maps.
+ *
+ * \sa transpose(), adjoint(), transposeInPlace() */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline void MatrixBase<Derived>::adjointInPlace() {
+  derived() = adjoint().eval();
+}
+
+#ifndef EIGEN_NO_DEBUG
+
+// The following is to detect aliasing problems in most common cases.
+
+namespace internal {
+
+template <bool DestIsTransposed, typename OtherDerived>
+struct check_transpose_aliasing_compile_time_selector {
+  enum { ret = bool(blas_traits<OtherDerived>::IsTransposed) != DestIsTransposed };
+};
+
+template <bool DestIsTransposed, typename BinOp, typename DerivedA, typename DerivedB>
+struct check_transpose_aliasing_compile_time_selector<DestIsTransposed, CwiseBinaryOp<BinOp, DerivedA, DerivedB> > {
+  enum {
+    ret = bool(blas_traits<DerivedA>::IsTransposed) != DestIsTransposed ||
+          bool(blas_traits<DerivedB>::IsTransposed) != DestIsTransposed
+  };
+};
+
+template <typename Scalar, bool DestIsTransposed, typename OtherDerived>
+struct check_transpose_aliasing_run_time_selector {
+  EIGEN_DEVICE_FUNC static bool run(const Scalar* dest, const OtherDerived& src) {
+    return (bool(blas_traits<OtherDerived>::IsTransposed) != DestIsTransposed) &&
+           (dest != 0 && dest == (const Scalar*)extract_data(src));
+  }
+};
+
+template <typename Scalar, bool DestIsTransposed, typename BinOp, typename DerivedA, typename DerivedB>
+struct check_transpose_aliasing_run_time_selector<Scalar, DestIsTransposed, CwiseBinaryOp<BinOp, DerivedA, DerivedB> > {
+  EIGEN_DEVICE_FUNC static bool run(const Scalar* dest, const CwiseBinaryOp<BinOp, DerivedA, DerivedB>& src) {
+    return ((blas_traits<DerivedA>::IsTransposed != DestIsTransposed) &&
+            (dest != 0 && dest == (const Scalar*)extract_data(src.lhs()))) ||
+           ((blas_traits<DerivedB>::IsTransposed != DestIsTransposed) &&
+            (dest != 0 && dest == (const Scalar*)extract_data(src.rhs())));
+  }
+};
+
+// the following selector, checkTransposeAliasing_impl, based on MightHaveTransposeAliasing,
+// is because when the condition controlling the assert is known at compile time, ICC emits a warning.
+// This is actually a good warning: in expressions that don't have any transposing, the condition is
+// known at compile time to be false, and using that, we can avoid generating the code of the assert again
+// and again for all these expressions that don't need it.
+
+template <typename Derived, typename OtherDerived,
+          bool MightHaveTransposeAliasing =
+              check_transpose_aliasing_compile_time_selector<blas_traits<Derived>::IsTransposed, OtherDerived>::ret>
+struct checkTransposeAliasing_impl {
+  EIGEN_DEVICE_FUNC static void run(const Derived& dst, const OtherDerived& other) {
+    eigen_assert(
+        (!check_transpose_aliasing_run_time_selector<typename Derived::Scalar, blas_traits<Derived>::IsTransposed,
+                                                     OtherDerived>::run(extract_data(dst), other)) &&
+        "aliasing detected during transposition, use transposeInPlace() "
+        "or evaluate the rhs into a temporary using .eval()");
+  }
+};
+
+template <typename Derived, typename OtherDerived>
+struct checkTransposeAliasing_impl<Derived, OtherDerived, false> {
+  EIGEN_DEVICE_FUNC static void run(const Derived&, const OtherDerived&) {}
+};
+
+template <typename Dst, typename Src>
+EIGEN_DEVICE_FUNC inline void check_for_aliasing(const Dst& dst, const Src& src) {
+  if ((!Dst::IsVectorAtCompileTime) && dst.rows() > 1 && dst.cols() > 1)
+    internal::checkTransposeAliasing_impl<Dst, Src>::run(dst, src);
+}
+
+}  // end namespace internal
+
+#endif  // EIGEN_NO_DEBUG
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_TRANSPOSE_H

include/Eigen/src/Core/Transpositions.h

@@ -0,0 +1,323 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2010-2011 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_TRANSPOSITIONS_H
+#define EIGEN_TRANSPOSITIONS_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+template <typename Derived>
+class TranspositionsBase {
+  typedef internal::traits<Derived> Traits;
+
+ public:
+  typedef typename Traits::IndicesType IndicesType;
+  typedef typename IndicesType::Scalar StorageIndex;
+  typedef Eigen::Index Index;  ///< \deprecated since Eigen 3.3
+
+  EIGEN_DEVICE_FUNC Derived& derived() { return *static_cast<Derived*>(this); }
+  EIGEN_DEVICE_FUNC const Derived& derived() const { return *static_cast<const Derived*>(this); }
+
+  /** Copies the \a other transpositions into \c *this */
+  template <typename OtherDerived>
+  Derived& operator=(const TranspositionsBase<OtherDerived>& other) {
+    indices() = other.indices();
+    return derived();
+  }
+
+  /** \returns the number of transpositions */
+  EIGEN_DEVICE_FUNC Index size() const { return indices().size(); }
+  /** \returns the number of rows of the equivalent permutation matrix */
+  EIGEN_DEVICE_FUNC Index rows() const { return indices().size(); }
+  /** \returns the number of columns of the equivalent permutation matrix */
+  EIGEN_DEVICE_FUNC Index cols() const { return indices().size(); }
+
+  /** Direct access to the underlying index vector */
+  EIGEN_DEVICE_FUNC inline const StorageIndex& coeff(Index i) const { return indices().coeff(i); }
+  /** Direct access to the underlying index vector */
+  inline StorageIndex& coeffRef(Index i) { return indices().coeffRef(i); }
+  /** Direct access to the underlying index vector */
+  inline const StorageIndex& operator()(Index i) const { return indices()(i); }
+  /** Direct access to the underlying index vector */
+  inline StorageIndex& operator()(Index i) { return indices()(i); }
+  /** Direct access to the underlying index vector */
+  inline const StorageIndex& operator[](Index i) const { return indices()(i); }
+  /** Direct access to the underlying index vector */
+  inline StorageIndex& operator[](Index i) { return indices()(i); }
+
+  /** const version of indices(). */
+  EIGEN_DEVICE_FUNC const IndicesType& indices() const { return derived().indices(); }
+  /** \returns a reference to the stored array representing the transpositions. */
+  EIGEN_DEVICE_FUNC IndicesType& indices() { return derived().indices(); }
+
+  /** Resizes to given size. */
+  inline void resize(Index newSize) { indices().resize(newSize); }
+
+  /** Sets \c *this to represents an identity transformation */
+  void setIdentity() {
+    for (StorageIndex i = 0; i < indices().size(); ++i) coeffRef(i) = i;
+  }
+
+  // FIXME: do we want such methods ?
+  // might be useful when the target matrix expression is complex, e.g.:
+  // object.matrix().block(..,..,..,..) = trans * object.matrix().block(..,..,..,..);
+  /*
+  template<typename MatrixType>
+  void applyForwardToRows(MatrixType& mat) const
+  {
+    for(Index k=0 ; k<size() ; ++k)
+      if(m_indices(k)!=k)
+        mat.row(k).swap(mat.row(m_indices(k)));
+  }
+
+  template<typename MatrixType>
+  void applyBackwardToRows(MatrixType& mat) const
+  {
+    for(Index k=size()-1 ; k>=0 ; --k)
+      if(m_indices(k)!=k)
+        mat.row(k).swap(mat.row(m_indices(k)));
+  }
+  */
+
+  /** \returns the inverse transformation */
+  inline Transpose<TranspositionsBase> inverse() const { return Transpose<TranspositionsBase>(derived()); }
+
+  /** \returns the transpose transformation */
+  inline Transpose<TranspositionsBase> transpose() const { return Transpose<TranspositionsBase>(derived()); }
+
+ protected:
+};
+
+namespace internal {
+template <int SizeAtCompileTime, int MaxSizeAtCompileTime, typename StorageIndex_>
+struct traits<Transpositions<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_> >
+    : traits<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_> > {
+  typedef Matrix<StorageIndex_, SizeAtCompileTime, 1, 0, MaxSizeAtCompileTime, 1> IndicesType;
+  typedef TranspositionsStorage StorageKind;
+};
+}  // namespace internal
+
+/** \class Transpositions
+ * \ingroup Core_Module
+ *
+ * \brief Represents a sequence of transpositions (row/column interchange)
+ *
+ * \tparam SizeAtCompileTime the number of transpositions, or Dynamic
+ * \tparam MaxSizeAtCompileTime the maximum number of transpositions, or Dynamic. This optional parameter defaults to
+ * SizeAtCompileTime. Most of the time, you should not have to specify it.
+ *
+ * This class represents a permutation transformation as a sequence of \em n transpositions
+ * \f$[T_{n-1} \ldots T_{i} \ldots T_{0}]\f$. It is internally stored as a vector of integers \c indices.
+ * Each transposition \f$ T_{i} \f$ applied on the left of a matrix (\f$ T_{i} M\f$) interchanges
+ * the rows \c i and \c indices[i] of the matrix \c M.
+ * A transposition applied on the right (e.g., \f$ M T_{i}\f$) yields a column interchange.
+ *
+ * Compared to the class PermutationMatrix, such a sequence of transpositions is what is
+ * computed during a decomposition with pivoting, and it is faster when applying the permutation in-place.
+ *
+ * To apply a sequence of transpositions to a matrix, simply use the operator * as in the following example:
+ * \code
+ * Transpositions tr;
+ * MatrixXf mat;
+ * mat = tr * mat;
+ * \endcode
+ * In this example, we detect that the matrix appears on both side, and so the transpositions
+ * are applied in-place without any temporary or extra copy.
+ *
+ * \sa class PermutationMatrix
+ */
+
+template <int SizeAtCompileTime, int MaxSizeAtCompileTime, typename StorageIndex_>
+class Transpositions
+    : public TranspositionsBase<Transpositions<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_> > {
+  typedef internal::traits<Transpositions> Traits;
+
+ public:
+  typedef TranspositionsBase<Transpositions> Base;
+  typedef typename Traits::IndicesType IndicesType;
+  typedef typename IndicesType::Scalar StorageIndex;
+
+  inline Transpositions() {}
+
+  /** Copy constructor. */
+  template <typename OtherDerived>
+  inline Transpositions(const TranspositionsBase<OtherDerived>& other) : m_indices(other.indices()) {}
+
+  /** Generic constructor from expression of the transposition indices. */
+  template <typename Other>
+  explicit inline Transpositions(const MatrixBase<Other>& indices) : m_indices(indices) {}
+
+  /** Copies the \a other transpositions into \c *this */
+  template <typename OtherDerived>
+  Transpositions& operator=(const TranspositionsBase<OtherDerived>& other) {
+    return Base::operator=(other);
+  }
+
+  /** Constructs an uninitialized permutation matrix of given size.
+   */
+  inline Transpositions(Index size) : m_indices(size) {}
+
+  /** const version of indices(). */
+  EIGEN_DEVICE_FUNC const IndicesType& indices() const { return m_indices; }
+  /** \returns a reference to the stored array representing the transpositions. */
+  EIGEN_DEVICE_FUNC IndicesType& indices() { return m_indices; }
+
+ protected:
+  IndicesType m_indices;
+};
+
+namespace internal {
+template <int SizeAtCompileTime, int MaxSizeAtCompileTime, typename StorageIndex_, int PacketAccess_>
+struct traits<Map<Transpositions<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_>, PacketAccess_> >
+    : traits<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_> > {
+  typedef Map<const Matrix<StorageIndex_, SizeAtCompileTime, 1, 0, MaxSizeAtCompileTime, 1>, PacketAccess_> IndicesType;
+  typedef StorageIndex_ StorageIndex;
+  typedef TranspositionsStorage StorageKind;
+};
+}  // namespace internal
+
+template <int SizeAtCompileTime, int MaxSizeAtCompileTime, typename StorageIndex_, int PacketAccess>
+class Map<Transpositions<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_>, PacketAccess>
+    : public TranspositionsBase<
+          Map<Transpositions<SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_>, PacketAccess> > {
+  typedef internal::traits<Map> Traits;
+
+ public:
+  typedef TranspositionsBase<Map> Base;
+  typedef typename Traits::IndicesType IndicesType;
+  typedef typename IndicesType::Scalar StorageIndex;
+
+  explicit inline Map(const StorageIndex* indicesPtr) : m_indices(indicesPtr) {}
+
+  inline Map(const StorageIndex* indicesPtr, Index size) : m_indices(indicesPtr, size) {}
+
+  /** Copies the \a other transpositions into \c *this */
+  template <typename OtherDerived>
+  Map& operator=(const TranspositionsBase<OtherDerived>& other) {
+    return Base::operator=(other);
+  }
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  /** This is a special case of the templated operator=. Its purpose is to
+   * prevent a default operator= from hiding the templated operator=.
+   */
+  Map& operator=(const Map& other) {
+    m_indices = other.m_indices;
+    return *this;
+  }
+#endif
+
+  /** const version of indices(). */
+  EIGEN_DEVICE_FUNC const IndicesType& indices() const { return m_indices; }
+
+  /** \returns a reference to the stored array representing the transpositions. */
+  EIGEN_DEVICE_FUNC IndicesType& indices() { return m_indices; }
+
+ protected:
+  IndicesType m_indices;
+};
+
+namespace internal {
+template <typename IndicesType_>
+struct traits<TranspositionsWrapper<IndicesType_> > : traits<PermutationWrapper<IndicesType_> > {
+  typedef TranspositionsStorage StorageKind;
+};
+}  // namespace internal
+
+template <typename IndicesType_>
+class TranspositionsWrapper : public TranspositionsBase<TranspositionsWrapper<IndicesType_> > {
+  typedef internal::traits<TranspositionsWrapper> Traits;
+
+ public:
+  typedef TranspositionsBase<TranspositionsWrapper> Base;
+  typedef typename Traits::IndicesType IndicesType;
+  typedef typename IndicesType::Scalar StorageIndex;
+
+  explicit inline TranspositionsWrapper(IndicesType& indices) : m_indices(indices) {}
+
+  /** Copies the \a other transpositions into \c *this */
+  template <typename OtherDerived>
+  TranspositionsWrapper& operator=(const TranspositionsBase<OtherDerived>& other) {
+    return Base::operator=(other);
+  }
+
+  /** const version of indices(). */
+  EIGEN_DEVICE_FUNC const IndicesType& indices() const { return m_indices; }
+
+  /** \returns a reference to the stored array representing the transpositions. */
+  EIGEN_DEVICE_FUNC IndicesType& indices() { return m_indices; }
+
+ protected:
+  typename IndicesType::Nested m_indices;
+};
+
+/** \returns the \a matrix with the \a transpositions applied to the columns.
+ */
+template <typename MatrixDerived, typename TranspositionsDerived>
+EIGEN_DEVICE_FUNC const Product<MatrixDerived, TranspositionsDerived, AliasFreeProduct> operator*(
+    const MatrixBase<MatrixDerived>& matrix, const TranspositionsBase<TranspositionsDerived>& transpositions) {
+  return Product<MatrixDerived, TranspositionsDerived, AliasFreeProduct>(matrix.derived(), transpositions.derived());
+}
+
+/** \returns the \a matrix with the \a transpositions applied to the rows.
+ */
+template <typename TranspositionsDerived, typename MatrixDerived>
+EIGEN_DEVICE_FUNC const Product<TranspositionsDerived, MatrixDerived, AliasFreeProduct> operator*(
+    const TranspositionsBase<TranspositionsDerived>& transpositions, const MatrixBase<MatrixDerived>& matrix) {
+  return Product<TranspositionsDerived, MatrixDerived, AliasFreeProduct>(transpositions.derived(), matrix.derived());
+}
+
+// Template partial specialization for transposed/inverse transpositions
+
+namespace internal {
+
+template <typename Derived>
+struct traits<Transpose<TranspositionsBase<Derived> > > : traits<Derived> {};
+
+}  // end namespace internal
+
+template <typename TranspositionsDerived>
+class Transpose<TranspositionsBase<TranspositionsDerived> > {
+  typedef TranspositionsDerived TranspositionType;
+  typedef typename TranspositionType::IndicesType IndicesType;
+
+ public:
+  explicit Transpose(const TranspositionType& t) : m_transpositions(t) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index size() const EIGEN_NOEXCEPT { return m_transpositions.size(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_transpositions.size(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_transpositions.size(); }
+
+  /** \returns the \a matrix with the inverse transpositions applied to the columns.
+   */
+  template <typename OtherDerived>
+  friend const Product<OtherDerived, Transpose, AliasFreeProduct> operator*(const MatrixBase<OtherDerived>& matrix,
+                                                                            const Transpose& trt) {
+    return Product<OtherDerived, Transpose, AliasFreeProduct>(matrix.derived(), trt);
+  }
+
+  /** \returns the \a matrix with the inverse transpositions applied to the rows.
+   */
+  template <typename OtherDerived>
+  const Product<Transpose, OtherDerived, AliasFreeProduct> operator*(const MatrixBase<OtherDerived>& matrix) const {
+    return Product<Transpose, OtherDerived, AliasFreeProduct>(*this, matrix.derived());
+  }
+
+  EIGEN_DEVICE_FUNC const TranspositionType& nestedExpression() const { return m_transpositions; }
+
+ protected:
+  const TranspositionType& m_transpositions;
+};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_TRANSPOSITIONS_H

include/Eigen/src/Core/TriangularMatrix.h

@@ -0,0 +1,900 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_TRIANGULARMATRIX_H
+#define EIGEN_TRIANGULARMATRIX_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <int Side, typename TriangularType, typename Rhs>
+struct triangular_solve_retval;
+
+}
+
+/** \class TriangularBase
+ * \ingroup Core_Module
+ *
+ * \brief Base class for triangular part in a matrix
+ */
+template <typename Derived>
+class TriangularBase : public EigenBase<Derived> {
+ public:
+  enum {
+    Mode = internal::traits<Derived>::Mode,
+    RowsAtCompileTime = internal::traits<Derived>::RowsAtCompileTime,
+    ColsAtCompileTime = internal::traits<Derived>::ColsAtCompileTime,
+    MaxRowsAtCompileTime = internal::traits<Derived>::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = internal::traits<Derived>::MaxColsAtCompileTime,
+
+    SizeAtCompileTime = (internal::size_of_xpr_at_compile_time<Derived>::ret),
+    /**< This is equal to the number of coefficients, i.e. the number of
+     * rows times the number of columns, or to \a Dynamic if this is not
+     * known at compile-time. \sa RowsAtCompileTime, ColsAtCompileTime */
+
+    MaxSizeAtCompileTime = internal::size_at_compile_time(internal::traits<Derived>::MaxRowsAtCompileTime,
+                                                          internal::traits<Derived>::MaxColsAtCompileTime)
+
+  };
+  typedef typename internal::traits<Derived>::Scalar Scalar;
+  typedef typename internal::traits<Derived>::StorageKind StorageKind;
+  typedef typename internal::traits<Derived>::StorageIndex StorageIndex;
+  typedef typename internal::traits<Derived>::FullMatrixType DenseMatrixType;
+  typedef DenseMatrixType DenseType;
+  typedef Derived const& Nested;
+
+  EIGEN_DEVICE_FUNC inline TriangularBase() {
+    eigen_assert(!((int(Mode) & int(UnitDiag)) && (int(Mode) & int(ZeroDiag))));
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return derived().rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return derived().cols(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index outerStride() const EIGEN_NOEXCEPT { return derived().outerStride(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index innerStride() const EIGEN_NOEXCEPT { return derived().innerStride(); }
+
+  // dummy resize function
+  EIGEN_DEVICE_FUNC void resize(Index rows, Index cols) {
+    EIGEN_UNUSED_VARIABLE(rows);
+    EIGEN_UNUSED_VARIABLE(cols);
+    eigen_assert(rows == this->rows() && cols == this->cols());
+  }
+
+  EIGEN_DEVICE_FUNC inline Scalar coeff(Index row, Index col) const { return derived().coeff(row, col); }
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index row, Index col) { return derived().coeffRef(row, col); }
+
+  /** \see MatrixBase::copyCoeff(row,col)
+   */
+  template <typename Other>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void copyCoeff(Index row, Index col, Other& other) {
+    derived().coeffRef(row, col) = other.coeff(row, col);
+  }
+
+  EIGEN_DEVICE_FUNC inline Scalar operator()(Index row, Index col) const {
+    check_coordinates(row, col);
+    return coeff(row, col);
+  }
+  EIGEN_DEVICE_FUNC inline Scalar& operator()(Index row, Index col) {
+    check_coordinates(row, col);
+    return coeffRef(row, col);
+  }
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  EIGEN_DEVICE_FUNC inline const Derived& derived() const { return *static_cast<const Derived*>(this); }
+  EIGEN_DEVICE_FUNC inline Derived& derived() { return *static_cast<Derived*>(this); }
+#endif  // not EIGEN_PARSED_BY_DOXYGEN
+
+  template <typename DenseDerived>
+  EIGEN_DEVICE_FUNC void evalTo(MatrixBase<DenseDerived>& other) const;
+  template <typename DenseDerived>
+  EIGEN_DEVICE_FUNC void evalToLazy(MatrixBase<DenseDerived>& other) const;
+
+  EIGEN_DEVICE_FUNC DenseMatrixType toDenseMatrix() const {
+    DenseMatrixType res(rows(), cols());
+    evalToLazy(res);
+    return res;
+  }
+
+ protected:
+  void check_coordinates(Index row, Index col) const {
+    EIGEN_ONLY_USED_FOR_DEBUG(row);
+    EIGEN_ONLY_USED_FOR_DEBUG(col);
+    eigen_assert(col >= 0 && col < cols() && row >= 0 && row < rows());
+    const int mode = int(Mode) & ~SelfAdjoint;
+    EIGEN_ONLY_USED_FOR_DEBUG(mode);
+    eigen_assert((mode == Upper && col >= row) || (mode == Lower && col <= row) ||
+                 ((mode == StrictlyUpper || mode == UnitUpper) && col > row) ||
+                 ((mode == StrictlyLower || mode == UnitLower) && col < row));
+  }
+
+#ifdef EIGEN_INTERNAL_DEBUGGING
+  void check_coordinates_internal(Index row, Index col) const { check_coordinates(row, col); }
+#else
+  void check_coordinates_internal(Index, Index) const {}
+#endif
+};
+
+/** \class TriangularView
+ * \ingroup Core_Module
+ *
+ * \brief Expression of a triangular part in a matrix
+ *
+ * \tparam MatrixType the type of the object in which we are taking the triangular part
+ * \tparam Mode the kind of triangular matrix expression to construct. Can be #Upper,
+ *             #Lower, #UnitUpper, #UnitLower, #StrictlyUpper, or #StrictlyLower.
+ *             This is in fact a bit field; it must have either #Upper or #Lower,
+ *             and additionally it may have #UnitDiag or #ZeroDiag or neither.
+ *
+ * This class represents a triangular part of a matrix, not necessarily square. Strictly speaking, for rectangular
+ * matrices one should speak of "trapezoid" parts. This class is the return type
+ * of MatrixBase::triangularView() and SparseMatrixBase::triangularView(), and most of the time this is the only way it
+ * is used.
+ *
+ * \sa MatrixBase::triangularView()
+ */
+namespace internal {
+template <typename MatrixType, unsigned int Mode_>
+struct traits<TriangularView<MatrixType, Mode_>> : traits<MatrixType> {
+  typedef typename ref_selector<MatrixType>::non_const_type MatrixTypeNested;
+  typedef std::remove_reference_t<MatrixTypeNested> MatrixTypeNestedNonRef;
+  typedef remove_all_t<MatrixTypeNested> MatrixTypeNestedCleaned;
+  typedef typename MatrixType::PlainObject FullMatrixType;
+  typedef MatrixType ExpressionType;
+  enum {
+    Mode = Mode_,
+    FlagsLvalueBit = is_lvalue<MatrixType>::value ? LvalueBit : 0,
+    Flags = (MatrixTypeNestedCleaned::Flags & (HereditaryBits | FlagsLvalueBit) &
+             (~(PacketAccessBit | DirectAccessBit | LinearAccessBit)))
+  };
+};
+}  // namespace internal
+
+template <typename MatrixType_, unsigned int Mode_, typename StorageKind>
+class TriangularViewImpl;
+
+template <typename MatrixType_, unsigned int Mode_>
+class TriangularView
+    : public TriangularViewImpl<MatrixType_, Mode_, typename internal::traits<MatrixType_>::StorageKind> {
+ public:
+  typedef TriangularViewImpl<MatrixType_, Mode_, typename internal::traits<MatrixType_>::StorageKind> Base;
+  typedef typename internal::traits<TriangularView>::Scalar Scalar;
+  typedef MatrixType_ MatrixType;
+
+ protected:
+  typedef typename internal::traits<TriangularView>::MatrixTypeNested MatrixTypeNested;
+  typedef typename internal::traits<TriangularView>::MatrixTypeNestedNonRef MatrixTypeNestedNonRef;
+
+  typedef internal::remove_all_t<typename MatrixType::ConjugateReturnType> MatrixConjugateReturnType;
+  typedef TriangularView<std::add_const_t<MatrixType>, Mode_> ConstTriangularView;
+
+ public:
+  typedef typename internal::traits<TriangularView>::StorageKind StorageKind;
+  typedef typename internal::traits<TriangularView>::MatrixTypeNestedCleaned NestedExpression;
+
+  enum {
+    Mode = Mode_,
+    Flags = internal::traits<TriangularView>::Flags,
+    TransposeMode = (int(Mode) & int(Upper) ? Lower : 0) | (int(Mode) & int(Lower) ? Upper : 0) |
+                    (int(Mode) & int(UnitDiag)) | (int(Mode) & int(ZeroDiag)),
+    IsVectorAtCompileTime = false
+  };
+
+  EIGEN_DEVICE_FUNC explicit inline TriangularView(MatrixType& matrix) : m_matrix(matrix) {}
+
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(TriangularView)
+
+  /** \copydoc EigenBase::rows() */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_matrix.rows(); }
+  /** \copydoc EigenBase::cols() */
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_matrix.cols(); }
+
+  /** \returns a const reference to the nested expression */
+  EIGEN_DEVICE_FUNC const NestedExpression& nestedExpression() const { return m_matrix; }
+
+  /** \returns a reference to the nested expression */
+  EIGEN_DEVICE_FUNC NestedExpression& nestedExpression() { return m_matrix; }
+
+  typedef TriangularView<const MatrixConjugateReturnType, Mode> ConjugateReturnType;
+  /** \sa MatrixBase::conjugate() const */
+  EIGEN_DEVICE_FUNC inline const ConjugateReturnType conjugate() const {
+    return ConjugateReturnType(m_matrix.conjugate());
+  }
+
+  /** \returns an expression of the complex conjugate of \c *this if Cond==true,
+   *           returns \c *this otherwise.
+   */
+  template <bool Cond>
+  EIGEN_DEVICE_FUNC inline std::conditional_t<Cond, ConjugateReturnType, ConstTriangularView> conjugateIf() const {
+    typedef std::conditional_t<Cond, ConjugateReturnType, ConstTriangularView> ReturnType;
+    return ReturnType(m_matrix.template conjugateIf<Cond>());
+  }
+
+  typedef TriangularView<const typename MatrixType::AdjointReturnType, TransposeMode> AdjointReturnType;
+  /** \sa MatrixBase::adjoint() const */
+  EIGEN_DEVICE_FUNC inline const AdjointReturnType adjoint() const { return AdjointReturnType(m_matrix.adjoint()); }
+
+  typedef TriangularView<typename MatrixType::TransposeReturnType, TransposeMode> TransposeReturnType;
+  /** \sa MatrixBase::transpose() */
+  template <class Dummy = int>
+  EIGEN_DEVICE_FUNC inline TransposeReturnType transpose(
+      std::enable_if_t<Eigen::internal::is_lvalue<MatrixType>::value, Dummy*> = nullptr) {
+    typename MatrixType::TransposeReturnType tmp(m_matrix);
+    return TransposeReturnType(tmp);
+  }
+
+  typedef TriangularView<const typename MatrixType::ConstTransposeReturnType, TransposeMode> ConstTransposeReturnType;
+  /** \sa MatrixBase::transpose() const */
+  EIGEN_DEVICE_FUNC inline const ConstTransposeReturnType transpose() const {
+    return ConstTransposeReturnType(m_matrix.transpose());
+  }
+
+  template <typename Other>
+  EIGEN_DEVICE_FUNC inline const Solve<TriangularView, Other> solve(const MatrixBase<Other>& other) const {
+    return Solve<TriangularView, Other>(*this, other.derived());
+  }
+
+// workaround MSVC ICE
+#if EIGEN_COMP_MSVC
+  template <int Side, typename Other>
+  EIGEN_DEVICE_FUNC inline const internal::triangular_solve_retval<Side, TriangularView, Other> solve(
+      const MatrixBase<Other>& other) const {
+    return Base::template solve<Side>(other);
+  }
+#else
+  using Base::solve;
+#endif
+
+  /** \returns a selfadjoint view of the referenced triangular part which must be either \c #Upper or \c #Lower.
+   *
+   * This is a shortcut for \code this->nestedExpression().selfadjointView<(*this)::Mode>() \endcode
+   * \sa MatrixBase::selfadjointView() */
+  EIGEN_DEVICE_FUNC SelfAdjointView<MatrixTypeNestedNonRef, Mode> selfadjointView() {
+    EIGEN_STATIC_ASSERT((Mode & (UnitDiag | ZeroDiag)) == 0, PROGRAMMING_ERROR);
+    return SelfAdjointView<MatrixTypeNestedNonRef, Mode>(m_matrix);
+  }
+
+  /** This is the const version of selfadjointView() */
+  EIGEN_DEVICE_FUNC const SelfAdjointView<MatrixTypeNestedNonRef, Mode> selfadjointView() const {
+    EIGEN_STATIC_ASSERT((Mode & (UnitDiag | ZeroDiag)) == 0, PROGRAMMING_ERROR);
+    return SelfAdjointView<MatrixTypeNestedNonRef, Mode>(m_matrix);
+  }
+
+  /** \returns the determinant of the triangular matrix
+   * \sa MatrixBase::determinant() */
+  EIGEN_DEVICE_FUNC Scalar determinant() const {
+    if (Mode & UnitDiag)
+      return 1;
+    else if (Mode & ZeroDiag)
+      return 0;
+    else
+      return m_matrix.diagonal().prod();
+  }
+
+ protected:
+  MatrixTypeNested m_matrix;
+};
+
+/** \ingroup Core_Module
+ *
+ * \brief Base class for a triangular part in a \b dense matrix
+ *
+ * This class is an abstract base class of class TriangularView, and objects of type TriangularViewImpl cannot be
+ * instantiated. It extends class TriangularView with additional methods which available for dense expressions only.
+ *
+ * \sa class TriangularView, MatrixBase::triangularView()
+ */
+template <typename MatrixType_, unsigned int Mode_>
+class TriangularViewImpl<MatrixType_, Mode_, Dense> : public TriangularBase<TriangularView<MatrixType_, Mode_>> {
+ public:
+  typedef TriangularView<MatrixType_, Mode_> TriangularViewType;
+
+  typedef TriangularBase<TriangularViewType> Base;
+  typedef typename internal::traits<TriangularViewType>::Scalar Scalar;
+
+  typedef MatrixType_ MatrixType;
+  typedef typename MatrixType::PlainObject DenseMatrixType;
+  typedef DenseMatrixType PlainObject;
+
+ public:
+  using Base::derived;
+  using Base::evalToLazy;
+
+  typedef typename internal::traits<TriangularViewType>::StorageKind StorageKind;
+
+  enum { Mode = Mode_, Flags = internal::traits<TriangularViewType>::Flags };
+
+  /** \returns the outer-stride of the underlying dense matrix
+   * \sa DenseCoeffsBase::outerStride() */
+  EIGEN_DEVICE_FUNC inline Index outerStride() const { return derived().nestedExpression().outerStride(); }
+  /** \returns the inner-stride of the underlying dense matrix
+   * \sa DenseCoeffsBase::innerStride() */
+  EIGEN_DEVICE_FUNC inline Index innerStride() const { return derived().nestedExpression().innerStride(); }
+
+  /** \sa MatrixBase::operator+=() */
+  template <typename Other>
+  EIGEN_DEVICE_FUNC TriangularViewType& operator+=(const DenseBase<Other>& other) {
+    internal::call_assignment_no_alias(derived(), other.derived(),
+                                       internal::add_assign_op<Scalar, typename Other::Scalar>());
+    return derived();
+  }
+  /** \sa MatrixBase::operator-=() */
+  template <typename Other>
+  EIGEN_DEVICE_FUNC TriangularViewType& operator-=(const DenseBase<Other>& other) {
+    internal::call_assignment_no_alias(derived(), other.derived(),
+                                       internal::sub_assign_op<Scalar, typename Other::Scalar>());
+    return derived();
+  }
+
+  /** \sa MatrixBase::operator*=() */
+  EIGEN_DEVICE_FUNC TriangularViewType& operator*=(const typename internal::traits<MatrixType>::Scalar& other) {
+    return *this = derived().nestedExpression() * other;
+  }
+  /** \sa DenseBase::operator/=() */
+  EIGEN_DEVICE_FUNC TriangularViewType& operator/=(const typename internal::traits<MatrixType>::Scalar& other) {
+    return *this = derived().nestedExpression() / other;
+  }
+
+  /** \sa MatrixBase::fill() */
+  EIGEN_DEVICE_FUNC void fill(const Scalar& value) { setConstant(value); }
+  /** \sa MatrixBase::setConstant() */
+  EIGEN_DEVICE_FUNC TriangularViewType& setConstant(const Scalar& value) {
+    return *this = MatrixType::Constant(derived().rows(), derived().cols(), value);
+  }
+  /** \sa MatrixBase::setZero() */
+  EIGEN_DEVICE_FUNC TriangularViewType& setZero() { return setConstant(Scalar(0)); }
+  /** \sa MatrixBase::setOnes() */
+  EIGEN_DEVICE_FUNC TriangularViewType& setOnes() { return setConstant(Scalar(1)); }
+
+  /** \sa MatrixBase::coeff()
+   * \warning the coordinates must fit into the referenced triangular part
+   */
+  EIGEN_DEVICE_FUNC inline Scalar coeff(Index row, Index col) const {
+    Base::check_coordinates_internal(row, col);
+    return derived().nestedExpression().coeff(row, col);
+  }
+
+  /** \sa MatrixBase::coeffRef()
+   * \warning the coordinates must fit into the referenced triangular part
+   */
+  EIGEN_DEVICE_FUNC inline Scalar& coeffRef(Index row, Index col) {
+    EIGEN_STATIC_ASSERT_LVALUE(TriangularViewType);
+    Base::check_coordinates_internal(row, col);
+    return derived().nestedExpression().coeffRef(row, col);
+  }
+
+  /** Assigns a triangular matrix to a triangular part of a dense matrix */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC TriangularViewType& operator=(const TriangularBase<OtherDerived>& other);
+
+  /** Shortcut for\code *this = other.other.triangularView<(*this)::Mode>() \endcode */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC TriangularViewType& operator=(const MatrixBase<OtherDerived>& other);
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+  EIGEN_DEVICE_FUNC TriangularViewType& operator=(const TriangularViewImpl& other) {
+    return *this = other.derived().nestedExpression();
+  }
+
+  template <typename OtherDerived>
+  /** \deprecated */
+  EIGEN_DEPRECATED EIGEN_DEVICE_FUNC void lazyAssign(const TriangularBase<OtherDerived>& other);
+
+  template <typename OtherDerived>
+  /** \deprecated */
+  EIGEN_DEPRECATED EIGEN_DEVICE_FUNC void lazyAssign(const MatrixBase<OtherDerived>& other);
+#endif
+
+  /** Efficient triangular matrix times vector/matrix product */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC const Product<TriangularViewType, OtherDerived> operator*(
+      const MatrixBase<OtherDerived>& rhs) const {
+    return Product<TriangularViewType, OtherDerived>(derived(), rhs.derived());
+  }
+
+  /** Efficient vector/matrix times triangular matrix product */
+  template <typename OtherDerived>
+  friend EIGEN_DEVICE_FUNC const Product<OtherDerived, TriangularViewType> operator*(
+      const MatrixBase<OtherDerived>& lhs, const TriangularViewImpl& rhs) {
+    return Product<OtherDerived, TriangularViewType>(lhs.derived(), rhs.derived());
+  }
+
+  /** \returns the product of the inverse of \c *this with \a other, \a *this being triangular.
+   *
+   * This function computes the inverse-matrix matrix product inverse(\c *this) * \a other if
+   * \a Side==OnTheLeft (the default), or the right-inverse-multiply  \a other * inverse(\c *this) if
+   * \a Side==OnTheRight.
+   *
+   * Note that the template parameter \c Side can be omitted, in which case \c Side==OnTheLeft
+   *
+   * The matrix \c *this must be triangular and invertible (i.e., all the coefficients of the
+   * diagonal must be non zero). It works as a forward (resp. backward) substitution if \c *this
+   * is an upper (resp. lower) triangular matrix.
+   *
+   * Example: \include Triangular_solve.cpp
+   * Output: \verbinclude Triangular_solve.out
+   *
+   * This function returns an expression of the inverse-multiply and can works in-place if it is assigned
+   * to the same matrix or vector \a other.
+   *
+   * For users coming from BLAS, this function (and more specifically solveInPlace()) offer
+   * all the operations supported by the \c *TRSV and \c *TRSM BLAS routines.
+   *
+   * \sa TriangularView::solveInPlace()
+   */
+  template <int Side, typename Other>
+  inline const internal::triangular_solve_retval<Side, TriangularViewType, Other> solve(
+      const MatrixBase<Other>& other) const;
+
+  /** "in-place" version of TriangularView::solve() where the result is written in \a other
+   *
+   * \warning The parameter is only marked 'const' to make the C++ compiler accept a temporary expression here.
+   * This function will const_cast it, so constness isn't honored here.
+   *
+   * Note that the template parameter \c Side can be omitted, in which case \c Side==OnTheLeft
+   *
+   * See TriangularView:solve() for the details.
+   */
+  template <int Side, typename OtherDerived>
+  EIGEN_DEVICE_FUNC void solveInPlace(const MatrixBase<OtherDerived>& other) const;
+
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC void solveInPlace(const MatrixBase<OtherDerived>& other) const {
+    return solveInPlace<OnTheLeft>(other);
+  }
+
+  /** Swaps the coefficients of the common triangular parts of two matrices */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC
+#ifdef EIGEN_PARSED_BY_DOXYGEN
+      void
+      swap(TriangularBase<OtherDerived>& other)
+#else
+      void
+      swap(TriangularBase<OtherDerived> const& other)
+#endif
+  {
+    EIGEN_STATIC_ASSERT_LVALUE(OtherDerived);
+    call_assignment(derived(), other.const_cast_derived(), internal::swap_assign_op<Scalar>());
+  }
+
+  /** Shortcut for \code (*this).swap(other.triangularView<(*this)::Mode>()) \endcode */
+  template <typename OtherDerived>
+  /** \deprecated */
+  EIGEN_DEPRECATED EIGEN_DEVICE_FUNC void swap(MatrixBase<OtherDerived> const& other) {
+    EIGEN_STATIC_ASSERT_LVALUE(OtherDerived);
+    call_assignment(derived(), other.const_cast_derived(), internal::swap_assign_op<Scalar>());
+  }
+
+  template <typename RhsType, typename DstType>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _solve_impl(const RhsType& rhs, DstType& dst) const {
+    if (!internal::is_same_dense(dst, rhs)) dst = rhs;
+    this->solveInPlace(dst);
+  }
+
+  template <typename ProductType>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TriangularViewType& _assignProduct(const ProductType& prod, const Scalar& alpha,
+                                                                           bool beta);
+
+ protected:
+  EIGEN_DEFAULT_COPY_CONSTRUCTOR(TriangularViewImpl)
+  EIGEN_DEFAULT_EMPTY_CONSTRUCTOR_AND_DESTRUCTOR(TriangularViewImpl)
+};
+
+/***************************************************************************
+ * Implementation of triangular evaluation/assignment
+ ***************************************************************************/
+
+#ifndef EIGEN_PARSED_BY_DOXYGEN
+// FIXME should we keep that possibility
+template <typename MatrixType, unsigned int Mode>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC inline TriangularView<MatrixType, Mode>& TriangularViewImpl<MatrixType, Mode, Dense>::operator=(
+    const MatrixBase<OtherDerived>& other) {
+  internal::call_assignment_no_alias(derived(), other.derived(),
+                                     internal::assign_op<Scalar, typename OtherDerived::Scalar>());
+  return derived();
+}
+
+// FIXME should we keep that possibility
+template <typename MatrixType, unsigned int Mode>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC void TriangularViewImpl<MatrixType, Mode, Dense>::lazyAssign(const MatrixBase<OtherDerived>& other) {
+  internal::call_assignment_no_alias(derived(), other.template triangularView<Mode>());
+}
+
+template <typename MatrixType, unsigned int Mode>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC inline TriangularView<MatrixType, Mode>& TriangularViewImpl<MatrixType, Mode, Dense>::operator=(
+    const TriangularBase<OtherDerived>& other) {
+  eigen_assert(Mode == int(OtherDerived::Mode));
+  internal::call_assignment(derived(), other.derived());
+  return derived();
+}
+
+template <typename MatrixType, unsigned int Mode>
+template <typename OtherDerived>
+EIGEN_DEVICE_FUNC void TriangularViewImpl<MatrixType, Mode, Dense>::lazyAssign(
+    const TriangularBase<OtherDerived>& other) {
+  eigen_assert(Mode == int(OtherDerived::Mode));
+  internal::call_assignment_no_alias(derived(), other.derived());
+}
+#endif
+
+/***************************************************************************
+ * Implementation of TriangularBase methods
+ ***************************************************************************/
+
+/** Assigns a triangular or selfadjoint matrix to a dense matrix.
+ * If the matrix is triangular, the opposite part is set to zero. */
+template <typename Derived>
+template <typename DenseDerived>
+EIGEN_DEVICE_FUNC void TriangularBase<Derived>::evalTo(MatrixBase<DenseDerived>& other) const {
+  evalToLazy(other.derived());
+}
+
+/***************************************************************************
+ * Implementation of TriangularView methods
+ ***************************************************************************/
+
+/***************************************************************************
+ * Implementation of MatrixBase methods
+ ***************************************************************************/
+
+/**
+ * \returns an expression of a triangular view extracted from the current matrix
+ *
+ * The parameter \a Mode can have the following values: \c #Upper, \c #StrictlyUpper, \c #UnitUpper,
+ * \c #Lower, \c #StrictlyLower, \c #UnitLower.
+ *
+ * Example: \include MatrixBase_triangularView.cpp
+ * Output: \verbinclude MatrixBase_triangularView.out
+ *
+ * \sa class TriangularView
+ */
+template <typename Derived>
+template <unsigned int Mode>
+EIGEN_DEVICE_FUNC typename MatrixBase<Derived>::template TriangularViewReturnType<Mode>::Type
+MatrixBase<Derived>::triangularView() {
+  return typename TriangularViewReturnType<Mode>::Type(derived());
+}
+
+/** This is the const version of MatrixBase::triangularView() */
+template <typename Derived>
+template <unsigned int Mode>
+EIGEN_DEVICE_FUNC typename MatrixBase<Derived>::template ConstTriangularViewReturnType<Mode>::Type
+MatrixBase<Derived>::triangularView() const {
+  return typename ConstTriangularViewReturnType<Mode>::Type(derived());
+}
+
+/** \returns true if *this is approximately equal to an upper triangular matrix,
+ *          within the precision given by \a prec.
+ *
+ * \sa isLowerTriangular()
+ */
+template <typename Derived>
+bool MatrixBase<Derived>::isUpperTriangular(const RealScalar& prec) const {
+  RealScalar maxAbsOnUpperPart = static_cast<RealScalar>(-1);
+  for (Index j = 0; j < cols(); ++j) {
+    Index maxi = numext::mini(j, rows() - 1);
+    for (Index i = 0; i <= maxi; ++i) {
+      RealScalar absValue = numext::abs(coeff(i, j));
+      if (absValue > maxAbsOnUpperPart) maxAbsOnUpperPart = absValue;
+    }
+  }
+  RealScalar threshold = maxAbsOnUpperPart * prec;
+  for (Index j = 0; j < cols(); ++j)
+    for (Index i = j + 1; i < rows(); ++i)
+      if (numext::abs(coeff(i, j)) > threshold) return false;
+  return true;
+}
+
+/** \returns true if *this is approximately equal to a lower triangular matrix,
+ *          within the precision given by \a prec.
+ *
+ * \sa isUpperTriangular()
+ */
+template <typename Derived>
+bool MatrixBase<Derived>::isLowerTriangular(const RealScalar& prec) const {
+  RealScalar maxAbsOnLowerPart = static_cast<RealScalar>(-1);
+  for (Index j = 0; j < cols(); ++j)
+    for (Index i = j; i < rows(); ++i) {
+      RealScalar absValue = numext::abs(coeff(i, j));
+      if (absValue > maxAbsOnLowerPart) maxAbsOnLowerPart = absValue;
+    }
+  RealScalar threshold = maxAbsOnLowerPart * prec;
+  for (Index j = 1; j < cols(); ++j) {
+    Index maxi = numext::mini(j, rows() - 1);
+    for (Index i = 0; i < maxi; ++i)
+      if (numext::abs(coeff(i, j)) > threshold) return false;
+  }
+  return true;
+}
+
+/***************************************************************************
+****************************************************************************
+* Evaluators and Assignment of triangular expressions
+***************************************************************************
+***************************************************************************/
+
+namespace internal {
+
+// TODO currently a triangular expression has the form TriangularView<.,.>
+//      in the future triangular-ness should be defined by the expression traits
+//      such that Transpose<TriangularView<.,.> > is valid. (currently TriangularBase::transpose() is overloaded to make
+//      it work)
+template <typename MatrixType, unsigned int Mode>
+struct evaluator_traits<TriangularView<MatrixType, Mode>> {
+  typedef typename storage_kind_to_evaluator_kind<typename MatrixType::StorageKind>::Kind Kind;
+  typedef typename glue_shapes<typename evaluator_traits<MatrixType>::Shape, TriangularShape>::type Shape;
+};
+
+template <typename MatrixType, unsigned int Mode>
+struct unary_evaluator<TriangularView<MatrixType, Mode>, IndexBased> : evaluator<internal::remove_all_t<MatrixType>> {
+  typedef TriangularView<MatrixType, Mode> XprType;
+  typedef evaluator<internal::remove_all_t<MatrixType>> Base;
+  EIGEN_DEVICE_FUNC unary_evaluator(const XprType& xpr) : Base(xpr.nestedExpression()) {}
+};
+
+// Additional assignment kinds:
+struct Triangular2Triangular {};
+struct Triangular2Dense {};
+struct Dense2Triangular {};
+
+template <typename Kernel, unsigned int Mode, int UnrollCount, bool ClearOpposite>
+struct triangular_assignment_loop;
+
+/** \internal Specialization of the dense assignment kernel for triangular matrices.
+ * The main difference is that the triangular, diagonal, and opposite parts are processed through three different
+ * functions. \tparam UpLo must be either Lower or Upper \tparam Mode must be either 0, UnitDiag, ZeroDiag, or
+ * SelfAdjoint
+ */
+template <int UpLo, int Mode, int SetOpposite, typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor,
+          int Version = Specialized>
+class triangular_dense_assignment_kernel
+    : public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, Version> {
+ protected:
+  typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, Version> Base;
+  typedef typename Base::DstXprType DstXprType;
+  typedef typename Base::SrcXprType SrcXprType;
+  using Base::m_dst;
+  using Base::m_functor;
+  using Base::m_src;
+
+ public:
+  typedef typename Base::DstEvaluatorType DstEvaluatorType;
+  typedef typename Base::SrcEvaluatorType SrcEvaluatorType;
+  typedef typename Base::Scalar Scalar;
+  typedef typename Base::AssignmentTraits AssignmentTraits;
+
+  EIGEN_DEVICE_FUNC triangular_dense_assignment_kernel(DstEvaluatorType& dst, const SrcEvaluatorType& src,
+                                                       const Functor& func, DstXprType& dstExpr)
+      : Base(dst, src, func, dstExpr) {}
+
+#ifdef EIGEN_INTERNAL_DEBUGGING
+  EIGEN_DEVICE_FUNC void assignCoeff(Index row, Index col) {
+    eigen_internal_assert(row != col);
+    Base::assignCoeff(row, col);
+  }
+#else
+  using Base::assignCoeff;
+#endif
+
+  EIGEN_DEVICE_FUNC void assignDiagonalCoeff(Index id) {
+    if (Mode == UnitDiag && SetOpposite)
+      m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(1));
+    else if (Mode == ZeroDiag && SetOpposite)
+      m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(0));
+    else if (Mode == 0)
+      Base::assignCoeff(id, id);
+  }
+
+  EIGEN_DEVICE_FUNC void assignOppositeCoeff(Index row, Index col) {
+    eigen_internal_assert(row != col);
+    if (SetOpposite) m_functor.assignCoeff(m_dst.coeffRef(row, col), Scalar(0));
+  }
+};
+
+template <int Mode, bool SetOpposite, typename DstXprType, typename SrcXprType, typename Functor>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_triangular_assignment_loop(DstXprType& dst, const SrcXprType& src,
+                                                                           const Functor& func) {
+  typedef evaluator<DstXprType> DstEvaluatorType;
+  typedef evaluator<SrcXprType> SrcEvaluatorType;
+
+  SrcEvaluatorType srcEvaluator(src);
+
+  Index dstRows = src.rows();
+  Index dstCols = src.cols();
+  if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
+  DstEvaluatorType dstEvaluator(dst);
+
+  typedef triangular_dense_assignment_kernel<Mode&(Lower | Upper), Mode&(UnitDiag | ZeroDiag | SelfAdjoint),
+                                             SetOpposite, DstEvaluatorType, SrcEvaluatorType, Functor>
+      Kernel;
+  Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
+
+  enum {
+    unroll = DstXprType::SizeAtCompileTime != Dynamic && SrcEvaluatorType::CoeffReadCost < HugeCost &&
+             DstXprType::SizeAtCompileTime *
+                     (int(DstEvaluatorType::CoeffReadCost) + int(SrcEvaluatorType::CoeffReadCost)) / 2 <=
+                 EIGEN_UNROLLING_LIMIT
+  };
+
+  triangular_assignment_loop<Kernel, Mode, unroll ? int(DstXprType::SizeAtCompileTime) : Dynamic, SetOpposite>::run(
+      kernel);
+}
+
+template <int Mode, bool SetOpposite, typename DstXprType, typename SrcXprType>
+EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_triangular_assignment_loop(DstXprType& dst, const SrcXprType& src) {
+  call_triangular_assignment_loop<Mode, SetOpposite>(
+      dst, src, internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>());
+}
+
+template <>
+struct AssignmentKind<TriangularShape, TriangularShape> {
+  typedef Triangular2Triangular Kind;
+};
+template <>
+struct AssignmentKind<DenseShape, TriangularShape> {
+  typedef Triangular2Dense Kind;
+};
+template <>
+struct AssignmentKind<TriangularShape, DenseShape> {
+  typedef Dense2Triangular Kind;
+};
+
+template <typename DstXprType, typename SrcXprType, typename Functor>
+struct Assignment<DstXprType, SrcXprType, Functor, Triangular2Triangular> {
+  EIGEN_DEVICE_FUNC static void run(DstXprType& dst, const SrcXprType& src, const Functor& func) {
+    eigen_assert(int(DstXprType::Mode) == int(SrcXprType::Mode));
+
+    call_triangular_assignment_loop<DstXprType::Mode, false>(dst, src, func);
+  }
+};
+
+template <typename DstXprType, typename SrcXprType, typename Functor>
+struct Assignment<DstXprType, SrcXprType, Functor, Triangular2Dense> {
+  EIGEN_DEVICE_FUNC static void run(DstXprType& dst, const SrcXprType& src, const Functor& func) {
+    call_triangular_assignment_loop<SrcXprType::Mode, (int(SrcXprType::Mode) & int(SelfAdjoint)) == 0>(dst, src, func);
+  }
+};
+
+template <typename DstXprType, typename SrcXprType, typename Functor>
+struct Assignment<DstXprType, SrcXprType, Functor, Dense2Triangular> {
+  EIGEN_DEVICE_FUNC static void run(DstXprType& dst, const SrcXprType& src, const Functor& func) {
+    call_triangular_assignment_loop<DstXprType::Mode, false>(dst, src, func);
+  }
+};
+
+template <typename Kernel, unsigned int Mode, int UnrollCount, bool SetOpposite>
+struct triangular_assignment_loop {
+  // FIXME: this is not very clean, perhaps this information should be provided by the kernel?
+  typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
+  typedef typename DstEvaluatorType::XprType DstXprType;
+
+  enum {
+    col = (UnrollCount - 1) / DstXprType::RowsAtCompileTime,
+    row = (UnrollCount - 1) % DstXprType::RowsAtCompileTime
+  };
+
+  typedef typename Kernel::Scalar Scalar;
+
+  EIGEN_DEVICE_FUNC static inline void run(Kernel& kernel) {
+    triangular_assignment_loop<Kernel, Mode, UnrollCount - 1, SetOpposite>::run(kernel);
+
+    if (row == col)
+      kernel.assignDiagonalCoeff(row);
+    else if (((Mode & Lower) && row > col) || ((Mode & Upper) && row < col))
+      kernel.assignCoeff(row, col);
+    else if (SetOpposite)
+      kernel.assignOppositeCoeff(row, col);
+  }
+};
+
+// prevent buggy user code from causing an infinite recursion
+template <typename Kernel, unsigned int Mode, bool SetOpposite>
+struct triangular_assignment_loop<Kernel, Mode, 0, SetOpposite> {
+  EIGEN_DEVICE_FUNC static inline void run(Kernel&) {}
+};
+
+// TODO: experiment with a recursive assignment procedure splitting the current
+//       triangular part into one rectangular and two triangular parts.
+
+template <typename Kernel, unsigned int Mode, bool SetOpposite>
+struct triangular_assignment_loop<Kernel, Mode, Dynamic, SetOpposite> {
+  typedef typename Kernel::Scalar Scalar;
+  EIGEN_DEVICE_FUNC static inline void run(Kernel& kernel) {
+    for (Index j = 0; j < kernel.cols(); ++j) {
+      Index maxi = numext::mini(j, kernel.rows());
+      Index i = 0;
+      if (((Mode & Lower) && SetOpposite) || (Mode & Upper)) {
+        for (; i < maxi; ++i)
+          if (Mode & Upper)
+            kernel.assignCoeff(i, j);
+          else
+            kernel.assignOppositeCoeff(i, j);
+      } else
+        i = maxi;
+
+      if (i < kernel.rows())  // then i==j
+        kernel.assignDiagonalCoeff(i++);
+
+      if (((Mode & Upper) && SetOpposite) || (Mode & Lower)) {
+        for (; i < kernel.rows(); ++i)
+          if (Mode & Lower)
+            kernel.assignCoeff(i, j);
+          else
+            kernel.assignOppositeCoeff(i, j);
+      }
+    }
+  }
+};
+
+}  // end namespace internal
+
+/** Assigns a triangular or selfadjoint matrix to a dense matrix.
+ * If the matrix is triangular, the opposite part is set to zero. */
+template <typename Derived>
+template <typename DenseDerived>
+EIGEN_DEVICE_FUNC void TriangularBase<Derived>::evalToLazy(MatrixBase<DenseDerived>& other) const {
+  other.derived().resize(this->rows(), this->cols());
+  internal::call_triangular_assignment_loop<Derived::Mode,
+                                            (int(Derived::Mode) & int(SelfAdjoint)) == 0 /* SetOpposite */>(
+      other.derived(), derived().nestedExpression());
+}
+
+namespace internal {
+
+// Triangular = Product
+template <typename DstXprType, typename Lhs, typename Rhs, typename Scalar>
+struct Assignment<DstXprType, Product<Lhs, Rhs, DefaultProduct>,
+                  internal::assign_op<Scalar, typename Product<Lhs, Rhs, DefaultProduct>::Scalar>, Dense2Triangular> {
+  typedef Product<Lhs, Rhs, DefaultProduct> SrcXprType;
+  static void run(DstXprType& dst, const SrcXprType& src,
+                  const internal::assign_op<Scalar, typename SrcXprType::Scalar>&) {
+    Index dstRows = src.rows();
+    Index dstCols = src.cols();
+    if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
+
+    dst._assignProduct(src, Scalar(1), false);
+  }
+};
+
+// Triangular += Product
+template <typename DstXprType, typename Lhs, typename Rhs, typename Scalar>
+struct Assignment<DstXprType, Product<Lhs, Rhs, DefaultProduct>,
+                  internal::add_assign_op<Scalar, typename Product<Lhs, Rhs, DefaultProduct>::Scalar>,
+                  Dense2Triangular> {
+  typedef Product<Lhs, Rhs, DefaultProduct> SrcXprType;
+  static void run(DstXprType& dst, const SrcXprType& src,
+                  const internal::add_assign_op<Scalar, typename SrcXprType::Scalar>&) {
+    dst._assignProduct(src, Scalar(1), true);
+  }
+};
+
+// Triangular -= Product
+template <typename DstXprType, typename Lhs, typename Rhs, typename Scalar>
+struct Assignment<DstXprType, Product<Lhs, Rhs, DefaultProduct>,
+                  internal::sub_assign_op<Scalar, typename Product<Lhs, Rhs, DefaultProduct>::Scalar>,
+                  Dense2Triangular> {
+  typedef Product<Lhs, Rhs, DefaultProduct> SrcXprType;
+  static void run(DstXprType& dst, const SrcXprType& src,
+                  const internal::sub_assign_op<Scalar, typename SrcXprType::Scalar>&) {
+    dst._assignProduct(src, Scalar(-1), true);
+  }
+};
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_TRIANGULARMATRIX_H

include/Eigen/src/Core/VectorBlock.h

@@ -0,0 +1,83 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_VECTORBLOCK_H
+#define EIGEN_VECTORBLOCK_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <typename VectorType, int Size>
+struct traits<VectorBlock<VectorType, Size> >
+    : public traits<Block<VectorType, traits<VectorType>::Flags & RowMajorBit ? 1 : Size,
+                          traits<VectorType>::Flags & RowMajorBit ? Size : 1> > {};
+}  // namespace internal
+
+/** \class VectorBlock
+ * \ingroup Core_Module
+ *
+ * \brief Expression of a fixed-size or dynamic-size sub-vector
+ *
+ * \tparam VectorType the type of the object in which we are taking a sub-vector
+ * \tparam Size size of the sub-vector we are taking at compile time (optional)
+ *
+ * This class represents an expression of either a fixed-size or dynamic-size sub-vector.
+ * It is the return type of DenseBase::segment(Index,Index) and DenseBase::segment<int>(Index) and
+ * most of the time this is the only way it is used.
+ *
+ * However, if you want to directly manipulate sub-vector expressions,
+ * for instance if you want to write a function returning such an expression, you
+ * will need to use this class.
+ *
+ * Here is an example illustrating the dynamic case:
+ * \include class_VectorBlock.cpp
+ * Output: \verbinclude class_VectorBlock.out
+ *
+ * \note Even though this expression has dynamic size, in the case where \a VectorType
+ * has fixed size, this expression inherits a fixed maximal size which means that evaluating
+ * it does not cause a dynamic memory allocation.
+ *
+ * Here is an example illustrating the fixed-size case:
+ * \include class_FixedVectorBlock.cpp
+ * Output: \verbinclude class_FixedVectorBlock.out
+ *
+ * \sa class Block, DenseBase::segment(Index,Index,Index,Index), DenseBase::segment(Index,Index)
+ */
+template <typename VectorType, int Size>
+class VectorBlock : public Block<VectorType, internal::traits<VectorType>::Flags & RowMajorBit ? 1 : Size,
+                                 internal::traits<VectorType>::Flags & RowMajorBit ? Size : 1> {
+  typedef Block<VectorType, internal::traits<VectorType>::Flags & RowMajorBit ? 1 : Size,
+                internal::traits<VectorType>::Flags & RowMajorBit ? Size : 1>
+      Base;
+  enum { IsColVector = !(internal::traits<VectorType>::Flags & RowMajorBit) };
+
+ public:
+  EIGEN_DENSE_PUBLIC_INTERFACE(VectorBlock)
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorBlock)
+  EIGEN_INHERIT_ASSIGNMENT_OPERATORS(VectorBlock)
+
+  /** Dynamic-size constructor
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE VectorBlock(VectorType& vector, Index start, Index size)
+      : Base(vector, IsColVector ? start : 0, IsColVector ? 0 : start, IsColVector ? size : 1, IsColVector ? 1 : size) {
+  }
+
+  /** Fixed-size constructor
+   */
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE VectorBlock(VectorType& vector, Index start)
+      : Base(vector, IsColVector ? start : 0, IsColVector ? 0 : start) {}
+};
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_VECTORBLOCK_H

include/Eigen/src/Core/VectorwiseOp.h

@@ -0,0 +1,713 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008-2019 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_PARTIAL_REDUX_H
+#define EIGEN_PARTIAL_REDUX_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+/** \class PartialReduxExpr
+ * \ingroup Core_Module
+ *
+ * \brief Generic expression of a partially reduxed matrix
+ *
+ * \tparam MatrixType the type of the matrix we are applying the redux operation
+ * \tparam MemberOp type of the member functor
+ * \tparam Direction indicates the direction of the redux (#Vertical or #Horizontal)
+ *
+ * This class represents an expression of a partial redux operator of a matrix.
+ * It is the return type of some VectorwiseOp functions,
+ * and most of the time this is the only way it is used.
+ *
+ * \sa class VectorwiseOp
+ */
+
+template <typename MatrixType, typename MemberOp, int Direction>
+class PartialReduxExpr;
+
+namespace internal {
+template <typename MatrixType, typename MemberOp, int Direction>
+struct traits<PartialReduxExpr<MatrixType, MemberOp, Direction> > : traits<MatrixType> {
+  typedef typename MemberOp::result_type Scalar;
+  typedef typename traits<MatrixType>::StorageKind StorageKind;
+  typedef typename traits<MatrixType>::XprKind XprKind;
+  typedef typename MatrixType::Scalar InputScalar;
+  enum {
+    RowsAtCompileTime = Direction == Vertical ? 1 : MatrixType::RowsAtCompileTime,
+    ColsAtCompileTime = Direction == Horizontal ? 1 : MatrixType::ColsAtCompileTime,
+    MaxRowsAtCompileTime = Direction == Vertical ? 1 : MatrixType::MaxRowsAtCompileTime,
+    MaxColsAtCompileTime = Direction == Horizontal ? 1 : MatrixType::MaxColsAtCompileTime,
+    Flags = RowsAtCompileTime == 1 ? RowMajorBit : 0,
+    TraversalSize = Direction == Vertical ? MatrixType::RowsAtCompileTime : MatrixType::ColsAtCompileTime
+  };
+};
+}  // namespace internal
+
+template <typename MatrixType, typename MemberOp, int Direction>
+class PartialReduxExpr : public internal::dense_xpr_base<PartialReduxExpr<MatrixType, MemberOp, Direction> >::type,
+                         internal::no_assignment_operator {
+ public:
+  typedef typename internal::dense_xpr_base<PartialReduxExpr>::type Base;
+  EIGEN_DENSE_PUBLIC_INTERFACE(PartialReduxExpr)
+
+  EIGEN_DEVICE_FUNC explicit PartialReduxExpr(const MatrixType& mat, const MemberOp& func = MemberOp())
+      : m_matrix(mat), m_functor(func) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT {
+    return (Direction == Vertical ? 1 : m_matrix.rows());
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT {
+    return (Direction == Horizontal ? 1 : m_matrix.cols());
+  }
+
+  EIGEN_DEVICE_FUNC typename MatrixType::Nested nestedExpression() const { return m_matrix; }
+
+  EIGEN_DEVICE_FUNC const MemberOp& functor() const { return m_functor; }
+
+ protected:
+  typename MatrixType::Nested m_matrix;
+  const MemberOp m_functor;
+};
+
+template <typename A, typename B>
+struct partial_redux_dummy_func;
+
+#define EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(MEMBER, COST, VECTORIZABLE, BINARYOP)              \
+  template <typename ResultType, typename Scalar>                                           \
+  struct member_##MEMBER {                                                                  \
+    typedef ResultType result_type;                                                         \
+    typedef BINARYOP<Scalar, Scalar> BinaryOp;                                              \
+    template <int Size>                                                                     \
+    struct Cost {                                                                           \
+      enum { value = COST };                                                                \
+    };                                                                                      \
+    enum { Vectorizable = VECTORIZABLE };                                                   \
+    template <typename XprType>                                                             \
+    EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ResultType operator()(const XprType& mat) const { \
+      return mat.MEMBER();                                                                  \
+    }                                                                                       \
+    BinaryOp binaryFunc() const { return BinaryOp(); }                                      \
+  }
+
+#define EIGEN_MEMBER_FUNCTOR(MEMBER, COST) EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(MEMBER, COST, 0, partial_redux_dummy_func)
+
+namespace internal {
+
+EIGEN_MEMBER_FUNCTOR(norm, (Size + 5) * NumTraits<Scalar>::MulCost + (Size - 1) * NumTraits<Scalar>::AddCost);
+EIGEN_MEMBER_FUNCTOR(stableNorm, (Size + 5) * NumTraits<Scalar>::MulCost + (Size - 1) * NumTraits<Scalar>::AddCost);
+EIGEN_MEMBER_FUNCTOR(blueNorm, (Size + 5) * NumTraits<Scalar>::MulCost + (Size - 1) * NumTraits<Scalar>::AddCost);
+EIGEN_MEMBER_FUNCTOR(hypotNorm, (Size - 1) * functor_traits<scalar_hypot_op<Scalar> >::Cost);
+EIGEN_MEMBER_FUNCTOR(all, (Size - 1) * NumTraits<Scalar>::AddCost);
+EIGEN_MEMBER_FUNCTOR(any, (Size - 1) * NumTraits<Scalar>::AddCost);
+EIGEN_MEMBER_FUNCTOR(count, (Size - 1) * NumTraits<Scalar>::AddCost);
+
+EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(sum, (Size - 1) * NumTraits<Scalar>::AddCost, 1, internal::scalar_sum_op);
+EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(minCoeff, (Size - 1) * NumTraits<Scalar>::AddCost, 1, internal::scalar_min_op);
+EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(maxCoeff, (Size - 1) * NumTraits<Scalar>::AddCost, 1, internal::scalar_max_op);
+EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR(prod, (Size - 1) * NumTraits<Scalar>::MulCost, 1, internal::scalar_product_op);
+
+template <int p, typename ResultType, typename Scalar>
+struct member_lpnorm {
+  typedef ResultType result_type;
+  enum { Vectorizable = 0 };
+  template <int Size>
+  struct Cost {
+    enum { value = (Size + 5) * NumTraits<Scalar>::MulCost + (Size - 1) * NumTraits<Scalar>::AddCost };
+  };
+  EIGEN_DEVICE_FUNC member_lpnorm() {}
+  template <typename XprType>
+  EIGEN_DEVICE_FUNC inline ResultType operator()(const XprType& mat) const {
+    return mat.template lpNorm<p>();
+  }
+};
+
+template <typename BinaryOpT, typename Scalar>
+struct member_redux {
+  typedef BinaryOpT BinaryOp;
+  typedef typename result_of<BinaryOp(const Scalar&, const Scalar&)>::type result_type;
+
+  enum { Vectorizable = functor_traits<BinaryOp>::PacketAccess };
+  template <int Size>
+  struct Cost {
+    enum { value = (Size - 1) * functor_traits<BinaryOp>::Cost };
+  };
+  EIGEN_DEVICE_FUNC explicit member_redux(const BinaryOp func) : m_functor(func) {}
+  template <typename Derived>
+  EIGEN_DEVICE_FUNC inline result_type operator()(const DenseBase<Derived>& mat) const {
+    return mat.redux(m_functor);
+  }
+  const BinaryOp& binaryFunc() const { return m_functor; }
+  const BinaryOp m_functor;
+};
+}  // namespace internal
+
+/** \class VectorwiseOp
+ * \ingroup Core_Module
+ *
+ * \brief Pseudo expression providing broadcasting and partial reduction operations
+ *
+ * \tparam ExpressionType the type of the object on which to do partial reductions
+ * \tparam Direction indicates whether to operate on columns (#Vertical) or rows (#Horizontal)
+ *
+ * This class represents a pseudo expression with broadcasting and partial reduction features.
+ * It is the return type of DenseBase::colwise() and DenseBase::rowwise()
+ * and most of the time this is the only way it is explicitly used.
+ *
+ * To understand the logic of rowwise/colwise expression, let's consider a generic case `A.colwise().foo()`
+ * where `foo` is any method of `VectorwiseOp`. This expression is equivalent to applying `foo()` to each
+ * column of `A` and then re-assemble the outputs in a matrix expression:
+ * \code [A.col(0).foo(), A.col(1).foo(), ..., A.col(A.cols()-1).foo()] \endcode
+ *
+ * Example: \include MatrixBase_colwise.cpp
+ * Output: \verbinclude MatrixBase_colwise.out
+ *
+ * The begin() and end() methods are obviously exceptions to the previous rule as they
+ * return STL-compatible begin/end iterators to the rows or columns of the nested expression.
+ * Typical use cases include for-range-loop and calls to STL algorithms:
+ *
+ * Example: \include MatrixBase_colwise_iterator_cxx11.cpp
+ * Output: \verbinclude MatrixBase_colwise_iterator_cxx11.out
+ *
+ * For a partial reduction on an empty input, some rules apply.
+ * For the sake of clarity, let's consider a vertical reduction:
+ *   - If the number of columns is zero, then a 1x0 row-major vector expression is returned.
+ *   - Otherwise, if the number of rows is zero, then
+ *       - a row vector of zeros is returned for sum-like reductions (sum, squaredNorm, norm, etc.)
+ *       - a row vector of ones is returned for a product reduction (e.g., <code>MatrixXd(n,0).colwise().prod()</code>)
+ *       - an assert is triggered for all other reductions (minCoeff,maxCoeff,redux(bin_op))
+ *
+ * \sa DenseBase::colwise(), DenseBase::rowwise(), class PartialReduxExpr
+ */
+template <typename ExpressionType, int Direction>
+class VectorwiseOp {
+ public:
+  typedef typename ExpressionType::Scalar Scalar;
+  typedef typename ExpressionType::RealScalar RealScalar;
+  typedef Eigen::Index Index;  ///< \deprecated since Eigen 3.3
+  typedef typename internal::ref_selector<ExpressionType>::non_const_type ExpressionTypeNested;
+  typedef internal::remove_all_t<ExpressionTypeNested> ExpressionTypeNestedCleaned;
+
+  template <template <typename OutScalar, typename InputScalar> class Functor, typename ReturnScalar = Scalar>
+  struct ReturnType {
+    typedef PartialReduxExpr<ExpressionType, Functor<ReturnScalar, Scalar>, Direction> Type;
+  };
+
+  template <typename BinaryOp>
+  struct ReduxReturnType {
+    typedef PartialReduxExpr<ExpressionType, internal::member_redux<BinaryOp, Scalar>, Direction> Type;
+  };
+
+  enum { isVertical = (Direction == Vertical) ? 1 : 0, isHorizontal = (Direction == Horizontal) ? 1 : 0 };
+
+ protected:
+  template <typename OtherDerived>
+  struct ExtendedType {
+    typedef Replicate<OtherDerived, isVertical ? 1 : ExpressionType::RowsAtCompileTime,
+                      isHorizontal ? 1 : ExpressionType::ColsAtCompileTime>
+        Type;
+  };
+
+  /** \internal
+   * Replicates a vector to match the size of \c *this */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC typename ExtendedType<OtherDerived>::Type extendedTo(const DenseBase<OtherDerived>& other) const {
+    EIGEN_STATIC_ASSERT(internal::check_implication(isVertical, OtherDerived::MaxColsAtCompileTime == 1),
+                        YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED)
+    EIGEN_STATIC_ASSERT(internal::check_implication(isHorizontal, OtherDerived::MaxRowsAtCompileTime == 1),
+                        YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED)
+    return typename ExtendedType<OtherDerived>::Type(other.derived(), isVertical ? 1 : m_matrix.rows(),
+                                                     isHorizontal ? 1 : m_matrix.cols());
+  }
+
+  template <typename OtherDerived>
+  struct OppositeExtendedType {
+    typedef Replicate<OtherDerived, isHorizontal ? 1 : ExpressionType::RowsAtCompileTime,
+                      isVertical ? 1 : ExpressionType::ColsAtCompileTime>
+        Type;
+  };
+
+  /** \internal
+   * Replicates a vector in the opposite direction to match the size of \c *this */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC typename OppositeExtendedType<OtherDerived>::Type extendedToOpposite(
+      const DenseBase<OtherDerived>& other) const {
+    EIGEN_STATIC_ASSERT(internal::check_implication(isHorizontal, OtherDerived::MaxColsAtCompileTime == 1),
+                        YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED)
+    EIGEN_STATIC_ASSERT(internal::check_implication(isVertical, OtherDerived::MaxRowsAtCompileTime == 1),
+                        YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED)
+    return typename OppositeExtendedType<OtherDerived>::Type(other.derived(), isHorizontal ? 1 : m_matrix.rows(),
+                                                             isVertical ? 1 : m_matrix.cols());
+  }
+
+ public:
+  EIGEN_DEVICE_FUNC explicit inline VectorwiseOp(ExpressionType& matrix) : m_matrix(matrix) {}
+
+  /** \internal */
+  EIGEN_DEVICE_FUNC inline const ExpressionType& _expression() const { return m_matrix; }
+
+#ifdef EIGEN_PARSED_BY_DOXYGEN
+  /** STL-like <a href="https://en.cppreference.com/w/cpp/named_req/RandomAccessIterator">RandomAccessIterator</a>
+   * iterator type over the columns or rows as returned by the begin() and end() methods.
+   */
+  random_access_iterator_type iterator;
+  /** This is the const version of iterator (aka read-only) */
+  random_access_iterator_type const_iterator;
+#else
+  typedef internal::subvector_stl_iterator<ExpressionType, DirectionType(Direction)> iterator;
+  typedef internal::subvector_stl_iterator<const ExpressionType, DirectionType(Direction)> const_iterator;
+  typedef internal::subvector_stl_reverse_iterator<ExpressionType, DirectionType(Direction)> reverse_iterator;
+  typedef internal::subvector_stl_reverse_iterator<const ExpressionType, DirectionType(Direction)>
+      const_reverse_iterator;
+#endif
+
+  /** returns an iterator to the first row (rowwise) or column (colwise) of the nested expression.
+   * \sa end(), cbegin()
+   */
+  iterator begin() { return iterator(m_matrix, 0); }
+  /** const version of begin() */
+  const_iterator begin() const { return const_iterator(m_matrix, 0); }
+  /** const version of begin() */
+  const_iterator cbegin() const { return const_iterator(m_matrix, 0); }
+
+  /** returns a reverse iterator to the last row (rowwise) or column (colwise) of the nested expression.
+   * \sa rend(), crbegin()
+   */
+  reverse_iterator rbegin() {
+    return reverse_iterator(m_matrix, m_matrix.template subVectors<DirectionType(Direction)>() - 1);
+  }
+  /** const version of rbegin() */
+  const_reverse_iterator rbegin() const {
+    return const_reverse_iterator(m_matrix, m_matrix.template subVectors<DirectionType(Direction)>() - 1);
+  }
+  /** const version of rbegin() */
+  const_reverse_iterator crbegin() const {
+    return const_reverse_iterator(m_matrix, m_matrix.template subVectors<DirectionType(Direction)>() - 1);
+  }
+
+  /** returns an iterator to the row (resp. column) following the last row (resp. column) of the nested expression
+   * \sa begin(), cend()
+   */
+  iterator end() { return iterator(m_matrix, m_matrix.template subVectors<DirectionType(Direction)>()); }
+  /** const version of end() */
+  const_iterator end() const {
+    return const_iterator(m_matrix, m_matrix.template subVectors<DirectionType(Direction)>());
+  }
+  /** const version of end() */
+  const_iterator cend() const {
+    return const_iterator(m_matrix, m_matrix.template subVectors<DirectionType(Direction)>());
+  }
+
+  /** returns a reverse iterator to the row (resp. column) before the first row (resp. column) of the nested expression
+   * \sa begin(), cend()
+   */
+  reverse_iterator rend() { return reverse_iterator(m_matrix, -1); }
+  /** const version of rend() */
+  const_reverse_iterator rend() const { return const_reverse_iterator(m_matrix, -1); }
+  /** const version of rend() */
+  const_reverse_iterator crend() const { return const_reverse_iterator(m_matrix, -1); }
+
+  /** \returns a row or column vector expression of \c *this reduxed by \a func
+   *
+   * The template parameter \a BinaryOp is the type of the functor
+   * of the custom redux operator. Note that func must be an associative operator.
+   *
+   * \warning the size along the reduction direction must be strictly positive,
+   *          otherwise an assertion is triggered.
+   *
+   * \sa class VectorwiseOp, DenseBase::colwise(), DenseBase::rowwise()
+   */
+  template <typename BinaryOp>
+  EIGEN_DEVICE_FUNC const typename ReduxReturnType<BinaryOp>::Type redux(const BinaryOp& func = BinaryOp()) const {
+    eigen_assert(redux_length() > 0 && "you are using an empty matrix");
+    return typename ReduxReturnType<BinaryOp>::Type(_expression(), internal::member_redux<BinaryOp, Scalar>(func));
+  }
+
+  typedef typename ReturnType<internal::member_minCoeff>::Type MinCoeffReturnType;
+  typedef typename ReturnType<internal::member_maxCoeff>::Type MaxCoeffReturnType;
+  typedef PartialReduxExpr<const CwiseUnaryOp<internal::scalar_abs2_op<Scalar>, const ExpressionTypeNestedCleaned>,
+                           internal::member_sum<RealScalar, RealScalar>, Direction>
+      SquaredNormReturnType;
+  typedef CwiseUnaryOp<internal::scalar_sqrt_op<RealScalar>, const SquaredNormReturnType> NormReturnType;
+  typedef typename ReturnType<internal::member_blueNorm, RealScalar>::Type BlueNormReturnType;
+  typedef typename ReturnType<internal::member_stableNorm, RealScalar>::Type StableNormReturnType;
+  typedef typename ReturnType<internal::member_hypotNorm, RealScalar>::Type HypotNormReturnType;
+  typedef typename ReturnType<internal::member_sum>::Type SumReturnType;
+  typedef EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE(SumReturnType, Scalar, quotient) MeanReturnType;
+  typedef typename ReturnType<internal::member_all, bool>::Type AllReturnType;
+  typedef typename ReturnType<internal::member_any, bool>::Type AnyReturnType;
+  typedef PartialReduxExpr<ExpressionType, internal::member_count<Index, Scalar>, Direction> CountReturnType;
+  typedef typename ReturnType<internal::member_prod>::Type ProdReturnType;
+  typedef Reverse<const ExpressionType, Direction> ConstReverseReturnType;
+  typedef Reverse<ExpressionType, Direction> ReverseReturnType;
+
+  template <int p>
+  struct LpNormReturnType {
+    typedef PartialReduxExpr<ExpressionType, internal::member_lpnorm<p, RealScalar, Scalar>, Direction> Type;
+  };
+
+  /** \returns a row (or column) vector expression of the smallest coefficient
+   * of each column (or row) of the referenced expression.
+   *
+   * \warning the size along the reduction direction must be strictly positive,
+   *          otherwise an assertion is triggered.
+   *
+   * \warning the result is undefined if \c *this contains NaN.
+   *
+   * Example: \include PartialRedux_minCoeff.cpp
+   * Output: \verbinclude PartialRedux_minCoeff.out
+   *
+   * \sa DenseBase::minCoeff() */
+  EIGEN_DEVICE_FUNC const MinCoeffReturnType minCoeff() const {
+    eigen_assert(redux_length() > 0 && "you are using an empty matrix");
+    return MinCoeffReturnType(_expression());
+  }
+
+  /** \returns a row (or column) vector expression of the largest coefficient
+   * of each column (or row) of the referenced expression.
+   *
+   * \warning the size along the reduction direction must be strictly positive,
+   *          otherwise an assertion is triggered.
+   *
+   * \warning the result is undefined if \c *this contains NaN.
+   *
+   * Example: \include PartialRedux_maxCoeff.cpp
+   * Output: \verbinclude PartialRedux_maxCoeff.out
+   *
+   * \sa DenseBase::maxCoeff() */
+  EIGEN_DEVICE_FUNC const MaxCoeffReturnType maxCoeff() const {
+    eigen_assert(redux_length() > 0 && "you are using an empty matrix");
+    return MaxCoeffReturnType(_expression());
+  }
+
+  /** \returns a row (or column) vector expression of the squared norm
+   * of each column (or row) of the referenced expression.
+   * This is a vector with real entries, even if the original matrix has complex entries.
+   *
+   * Example: \include PartialRedux_squaredNorm.cpp
+   * Output: \verbinclude PartialRedux_squaredNorm.out
+   *
+   * \sa DenseBase::squaredNorm() */
+  EIGEN_DEVICE_FUNC const SquaredNormReturnType squaredNorm() const {
+    return SquaredNormReturnType(m_matrix.cwiseAbs2());
+  }
+
+  /** \returns a row (or column) vector expression of the norm
+   * of each column (or row) of the referenced expression.
+   * This is a vector with real entries, even if the original matrix has complex entries.
+   *
+   * Example: \include PartialRedux_norm.cpp
+   * Output: \verbinclude PartialRedux_norm.out
+   *
+   * \sa DenseBase::norm() */
+  EIGEN_DEVICE_FUNC const NormReturnType norm() const { return NormReturnType(squaredNorm()); }
+
+  /** \returns a row (or column) vector expression of the norm
+   * of each column (or row) of the referenced expression.
+   * This is a vector with real entries, even if the original matrix has complex entries.
+   *
+   * Example: \include PartialRedux_norm.cpp
+   * Output: \verbinclude PartialRedux_norm.out
+   *
+   * \sa DenseBase::norm() */
+  template <int p>
+  EIGEN_DEVICE_FUNC const typename LpNormReturnType<p>::Type lpNorm() const {
+    return typename LpNormReturnType<p>::Type(_expression());
+  }
+
+  /** \returns a row (or column) vector expression of the norm
+   * of each column (or row) of the referenced expression, using
+   * Blue's algorithm.
+   * This is a vector with real entries, even if the original matrix has complex entries.
+   *
+   * \sa DenseBase::blueNorm() */
+  EIGEN_DEVICE_FUNC const BlueNormReturnType blueNorm() const { return BlueNormReturnType(_expression()); }
+
+  /** \returns a row (or column) vector expression of the norm
+   * of each column (or row) of the referenced expression, avoiding
+   * underflow and overflow.
+   * This is a vector with real entries, even if the original matrix has complex entries.
+   *
+   * \sa DenseBase::stableNorm() */
+  EIGEN_DEVICE_FUNC const StableNormReturnType stableNorm() const { return StableNormReturnType(_expression()); }
+
+  /** \returns a row (or column) vector expression of the norm
+   * of each column (or row) of the referenced expression, avoiding
+   * underflow and overflow using a concatenation of hypot() calls.
+   * This is a vector with real entries, even if the original matrix has complex entries.
+   *
+   * \sa DenseBase::hypotNorm() */
+  EIGEN_DEVICE_FUNC const HypotNormReturnType hypotNorm() const { return HypotNormReturnType(_expression()); }
+
+  /** \returns a row (or column) vector expression of the sum
+   * of each column (or row) of the referenced expression.
+   *
+   * Example: \include PartialRedux_sum.cpp
+   * Output: \verbinclude PartialRedux_sum.out
+   *
+   * \sa DenseBase::sum() */
+  EIGEN_DEVICE_FUNC const SumReturnType sum() const { return SumReturnType(_expression()); }
+
+  /** \returns a row (or column) vector expression of the mean
+   * of each column (or row) of the referenced expression.
+   *
+   * \sa DenseBase::mean() */
+  EIGEN_DEVICE_FUNC const MeanReturnType mean() const {
+    return sum() / Scalar(Direction == Vertical ? m_matrix.rows() : m_matrix.cols());
+  }
+
+  /** \returns a row (or column) vector expression representing
+   * whether \b all coefficients of each respective column (or row) are \c true.
+   * This expression can be assigned to a vector with entries of type \c bool.
+   *
+   * \sa DenseBase::all() */
+  EIGEN_DEVICE_FUNC const AllReturnType all() const { return AllReturnType(_expression()); }
+
+  /** \returns a row (or column) vector expression representing
+   * whether \b at \b least one coefficient of each respective column (or row) is \c true.
+   * This expression can be assigned to a vector with entries of type \c bool.
+   *
+   * \sa DenseBase::any() */
+  EIGEN_DEVICE_FUNC const AnyReturnType any() const { return AnyReturnType(_expression()); }
+
+  /** \returns a row (or column) vector expression representing
+   * the number of \c true coefficients of each respective column (or row).
+   * This expression can be assigned to a vector whose entries have the same type as is used to
+   * index entries of the original matrix; for dense matrices, this is \c std::ptrdiff_t .
+   *
+   * Example: \include PartialRedux_count.cpp
+   * Output: \verbinclude PartialRedux_count.out
+   *
+   * \sa DenseBase::count() */
+  EIGEN_DEVICE_FUNC const CountReturnType count() const { return CountReturnType(_expression()); }
+
+  /** \returns a row (or column) vector expression of the product
+   * of each column (or row) of the referenced expression.
+   *
+   * Example: \include PartialRedux_prod.cpp
+   * Output: \verbinclude PartialRedux_prod.out
+   *
+   * \sa DenseBase::prod() */
+  EIGEN_DEVICE_FUNC const ProdReturnType prod() const { return ProdReturnType(_expression()); }
+
+  /** \returns a matrix expression
+   * where each column (or row) are reversed.
+   *
+   * Example: \include Vectorwise_reverse.cpp
+   * Output: \verbinclude Vectorwise_reverse.out
+   *
+   * \sa DenseBase::reverse() */
+  EIGEN_DEVICE_FUNC const ConstReverseReturnType reverse() const { return ConstReverseReturnType(_expression()); }
+
+  /** \returns a writable matrix expression
+   * where each column (or row) are reversed.
+   *
+   * \sa reverse() const */
+  EIGEN_DEVICE_FUNC ReverseReturnType reverse() { return ReverseReturnType(_expression()); }
+
+  typedef Replicate<ExpressionType, (isVertical ? Dynamic : 1), (isHorizontal ? Dynamic : 1)> ReplicateReturnType;
+  EIGEN_DEVICE_FUNC const ReplicateReturnType replicate(Index factor) const;
+
+  /**
+   * \return an expression of the replication of each column (or row) of \c *this
+   *
+   * Example: \include DirectionWise_replicate.cpp
+   * Output: \verbinclude DirectionWise_replicate.out
+   *
+   * \sa VectorwiseOp::replicate(Index), DenseBase::replicate(), class Replicate
+   */
+  // NOTE implemented here because of sunstudio's compilation errors
+  // isVertical*Factor+isHorizontal instead of (isVertical?Factor:1) to handle CUDA bug with ternary operator
+  template <int Factor>
+  const Replicate<ExpressionType, isVertical * Factor + isHorizontal,
+                  isHorizontal * Factor + isVertical> EIGEN_DEVICE_FUNC
+  replicate(Index factor = Factor) const {
+    return Replicate<ExpressionType, (isVertical ? Factor : 1), (isHorizontal ? Factor : 1)>(
+        _expression(), isVertical ? factor : 1, isHorizontal ? factor : 1);
+  }
+
+  /////////// Artithmetic operators ///////////
+
+  /** Copies the vector \a other to each subvector of \c *this */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC ExpressionType& operator=(const DenseBase<OtherDerived>& other) {
+    EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived)
+    EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived)
+    // eigen_assert((m_matrix.isNull()) == (other.isNull())); FIXME
+    return m_matrix = extendedTo(other.derived());
+  }
+
+  /** Adds the vector \a other to each subvector of \c *this */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC ExpressionType& operator+=(const DenseBase<OtherDerived>& other) {
+    EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived)
+    EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived)
+    return m_matrix += extendedTo(other.derived());
+  }
+
+  /** Subtracts the vector \a other to each subvector of \c *this */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC ExpressionType& operator-=(const DenseBase<OtherDerived>& other) {
+    EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived)
+    EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived)
+    return m_matrix -= extendedTo(other.derived());
+  }
+
+  /** Multiplies each subvector of \c *this by the vector \a other */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC ExpressionType& operator*=(const DenseBase<OtherDerived>& other) {
+    EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived)
+    EIGEN_STATIC_ASSERT_ARRAYXPR(ExpressionType)
+    EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived)
+    m_matrix *= extendedTo(other.derived());
+    return m_matrix;
+  }
+
+  /** Divides each subvector of \c *this by the vector \a other */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC ExpressionType& operator/=(const DenseBase<OtherDerived>& other) {
+    EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived)
+    EIGEN_STATIC_ASSERT_ARRAYXPR(ExpressionType)
+    EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived)
+    m_matrix /= extendedTo(other.derived());
+    return m_matrix;
+  }
+
+  /** Returns the expression of the sum of the vector \a other to each subvector of \c *this */
+  template <typename OtherDerived>
+  EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
+      CwiseBinaryOp<internal::scalar_sum_op<Scalar, typename OtherDerived::Scalar>, const ExpressionTypeNestedCleaned,
+                    const typename ExtendedType<OtherDerived>::Type>
+      operator+(const DenseBase<OtherDerived>& other) const {
+    EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived)
+    EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived)
+    return m_matrix + extendedTo(other.derived());
+  }
+
+  /** Returns the expression of the difference between each subvector of \c *this and the vector \a other */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC CwiseBinaryOp<internal::scalar_difference_op<Scalar, typename OtherDerived::Scalar>,
+                                  const ExpressionTypeNestedCleaned, const typename ExtendedType<OtherDerived>::Type>
+  operator-(const DenseBase<OtherDerived>& other) const {
+    EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived)
+    EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived)
+    return m_matrix - extendedTo(other.derived());
+  }
+
+  /** Returns the expression where each subvector is the product of the vector \a other
+   * by the corresponding subvector of \c *this */
+  template <typename OtherDerived>
+  EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
+      CwiseBinaryOp<internal::scalar_product_op<Scalar>, const ExpressionTypeNestedCleaned,
+                    const typename ExtendedType<OtherDerived>::Type> EIGEN_DEVICE_FUNC
+      operator*(const DenseBase<OtherDerived>& other) const {
+    EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived)
+    EIGEN_STATIC_ASSERT_ARRAYXPR(ExpressionType)
+    EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived)
+    return m_matrix * extendedTo(other.derived());
+  }
+
+  /** Returns the expression where each subvector is the quotient of the corresponding
+   * subvector of \c *this by the vector \a other */
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC CwiseBinaryOp<internal::scalar_quotient_op<Scalar>, const ExpressionTypeNestedCleaned,
+                                  const typename ExtendedType<OtherDerived>::Type>
+  operator/(const DenseBase<OtherDerived>& other) const {
+    EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived)
+    EIGEN_STATIC_ASSERT_ARRAYXPR(ExpressionType)
+    EIGEN_STATIC_ASSERT_SAME_XPR_KIND(ExpressionType, OtherDerived)
+    return m_matrix / extendedTo(other.derived());
+  }
+
+  /** \returns an expression where each column (or row) of the referenced matrix are normalized.
+   * The referenced matrix is \b not modified.
+   * \sa MatrixBase::normalized(), normalize()
+   */
+  EIGEN_DEVICE_FUNC CwiseBinaryOp<internal::scalar_quotient_op<Scalar>, const ExpressionTypeNestedCleaned,
+                                  const typename OppositeExtendedType<NormReturnType>::Type>
+  normalized() const {
+    return m_matrix.cwiseQuotient(extendedToOpposite(this->norm()));
+  }
+
+  /** Normalize in-place each row or columns of the referenced matrix.
+   * \sa MatrixBase::normalize(), normalized()
+   */
+  EIGEN_DEVICE_FUNC void normalize() { m_matrix = this->normalized(); }
+
+  EIGEN_DEVICE_FUNC inline void reverseInPlace();
+
+  /////////// Geometry module ///////////
+
+  typedef Homogeneous<ExpressionType, Direction> HomogeneousReturnType;
+  EIGEN_DEVICE_FUNC HomogeneousReturnType homogeneous() const;
+
+  typedef typename ExpressionType::PlainObject CrossReturnType;
+  template <typename OtherDerived>
+  EIGEN_DEVICE_FUNC const CrossReturnType cross(const MatrixBase<OtherDerived>& other) const;
+
+  enum {
+    HNormalized_Size = Direction == Vertical ? internal::traits<ExpressionType>::RowsAtCompileTime
+                                             : internal::traits<ExpressionType>::ColsAtCompileTime,
+    HNormalized_SizeMinusOne = HNormalized_Size == Dynamic ? Dynamic : HNormalized_Size - 1
+  };
+  typedef Block<const ExpressionType,
+                Direction == Vertical ? int(HNormalized_SizeMinusOne)
+                                      : int(internal::traits<ExpressionType>::RowsAtCompileTime),
+                Direction == Horizontal ? int(HNormalized_SizeMinusOne)
+                                        : int(internal::traits<ExpressionType>::ColsAtCompileTime)>
+      HNormalized_Block;
+  typedef Block<const ExpressionType,
+                Direction == Vertical ? 1 : int(internal::traits<ExpressionType>::RowsAtCompileTime),
+                Direction == Horizontal ? 1 : int(internal::traits<ExpressionType>::ColsAtCompileTime)>
+      HNormalized_Factors;
+  typedef CwiseBinaryOp<internal::scalar_quotient_op<typename internal::traits<ExpressionType>::Scalar>,
+                        const HNormalized_Block,
+                        const Replicate<HNormalized_Factors, Direction == Vertical ? HNormalized_SizeMinusOne : 1,
+                                        Direction == Horizontal ? HNormalized_SizeMinusOne : 1> >
+      HNormalizedReturnType;
+
+  EIGEN_DEVICE_FUNC const HNormalizedReturnType hnormalized() const;
+
+#ifdef EIGEN_VECTORWISEOP_PLUGIN
+#include EIGEN_VECTORWISEOP_PLUGIN
+#endif
+
+ protected:
+  EIGEN_DEVICE_FUNC Index redux_length() const { return Direction == Vertical ? m_matrix.rows() : m_matrix.cols(); }
+  ExpressionTypeNested m_matrix;
+};
+
+// const colwise moved to DenseBase.h due to CUDA compiler bug
+
+/** \returns a writable VectorwiseOp wrapper of *this providing additional partial reduction operations
+ *
+ * \sa rowwise(), class VectorwiseOp, \ref TutorialReductionsVisitorsBroadcasting
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline typename DenseBase<Derived>::ColwiseReturnType DenseBase<Derived>::colwise() {
+  return ColwiseReturnType(derived());
+}
+
+// const rowwise moved to DenseBase.h due to CUDA compiler bug
+
+/** \returns a writable VectorwiseOp wrapper of *this providing additional partial reduction operations
+ *
+ * \sa colwise(), class VectorwiseOp, \ref TutorialReductionsVisitorsBroadcasting
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline typename DenseBase<Derived>::RowwiseReturnType DenseBase<Derived>::rowwise() {
+  return RowwiseReturnType(derived());
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_PARTIAL_REDUX_H

include/Eigen/src/Core/Visitor.h

@@ -0,0 +1,789 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_VISITOR_H
+#define EIGEN_VISITOR_H
+
+// IWYU pragma: private
+#include "./InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Visitor, typename Derived, int UnrollCount,
+          bool Vectorize = (Derived::PacketAccess && functor_traits<Visitor>::PacketAccess), bool LinearAccess = false,
+          bool ShortCircuitEvaluation = false>
+struct visitor_impl;
+
+template <typename Visitor, bool ShortCircuitEvaluation = false>
+struct short_circuit_eval_impl {
+  // if short circuit evaluation is not used, do nothing
+  static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const Visitor&) { return false; }
+};
+template <typename Visitor>
+struct short_circuit_eval_impl<Visitor, true> {
+  // if short circuit evaluation is used, check the visitor
+  static EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const Visitor& visitor) {
+    return visitor.done();
+  }
+};
+
+// unrolled inner-outer traversal
+template <typename Visitor, typename Derived, int UnrollCount, bool Vectorize, bool ShortCircuitEvaluation>
+struct visitor_impl<Visitor, Derived, UnrollCount, Vectorize, false, ShortCircuitEvaluation> {
+  // don't use short circuit evaluation for unrolled version
+  using Scalar = typename Derived::Scalar;
+  using Packet = typename packet_traits<Scalar>::type;
+  static constexpr bool RowMajor = Derived::IsRowMajor;
+  static constexpr int RowsAtCompileTime = Derived::RowsAtCompileTime;
+  static constexpr int ColsAtCompileTime = Derived::ColsAtCompileTime;
+  static constexpr int PacketSize = packet_traits<Scalar>::size;
+
+  static constexpr bool CanVectorize(int K) {
+    constexpr int InnerSizeAtCompileTime = RowMajor ? ColsAtCompileTime : RowsAtCompileTime;
+    if (InnerSizeAtCompileTime < PacketSize) return false;
+    return Vectorize && (InnerSizeAtCompileTime - (K % InnerSizeAtCompileTime) >= PacketSize);
+  }
+
+  template <int K = 0, bool Empty = (K == UnrollCount), std::enable_if_t<Empty, bool> = true>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived&, Visitor&) {}
+
+  template <int K = 0, bool Empty = (K == UnrollCount), bool Initialize = (K == 0), bool DoVectorOp = CanVectorize(K),
+            std::enable_if_t<!Empty && Initialize && !DoVectorOp, bool> = true>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    visitor.init(mat.coeff(0, 0), 0, 0);
+    run<1>(mat, visitor);
+  }
+
+  template <int K = 0, bool Empty = (K == UnrollCount), bool Initialize = (K == 0), bool DoVectorOp = CanVectorize(K),
+            std::enable_if_t<!Empty && !Initialize && !DoVectorOp, bool> = true>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    static constexpr int R = RowMajor ? (K / ColsAtCompileTime) : (K % RowsAtCompileTime);
+    static constexpr int C = RowMajor ? (K % ColsAtCompileTime) : (K / RowsAtCompileTime);
+    visitor(mat.coeff(R, C), R, C);
+    run<K + 1>(mat, visitor);
+  }
+
+  template <int K = 0, bool Empty = (K == UnrollCount), bool Initialize = (K == 0), bool DoVectorOp = CanVectorize(K),
+            std::enable_if_t<!Empty && Initialize && DoVectorOp, bool> = true>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    Packet P = mat.template packet<Packet>(0, 0);
+    visitor.initpacket(P, 0, 0);
+    run<PacketSize>(mat, visitor);
+  }
+
+  template <int K = 0, bool Empty = (K == UnrollCount), bool Initialize = (K == 0), bool DoVectorOp = CanVectorize(K),
+            std::enable_if_t<!Empty && !Initialize && DoVectorOp, bool> = true>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    static constexpr int R = RowMajor ? (K / ColsAtCompileTime) : (K % RowsAtCompileTime);
+    static constexpr int C = RowMajor ? (K % ColsAtCompileTime) : (K / RowsAtCompileTime);
+    Packet P = mat.template packet<Packet>(R, C);
+    visitor.packet(P, R, C);
+    run<K + PacketSize>(mat, visitor);
+  }
+};
+
+// unrolled linear traversal
+template <typename Visitor, typename Derived, int UnrollCount, bool Vectorize, bool ShortCircuitEvaluation>
+struct visitor_impl<Visitor, Derived, UnrollCount, Vectorize, true, ShortCircuitEvaluation> {
+  // don't use short circuit evaluation for unrolled version
+  using Scalar = typename Derived::Scalar;
+  using Packet = typename packet_traits<Scalar>::type;
+  static constexpr int PacketSize = packet_traits<Scalar>::size;
+
+  static constexpr bool CanVectorize(int K) { return Vectorize && ((UnrollCount - K) >= PacketSize); }
+
+  // empty
+  template <int K = 0, bool Empty = (K == UnrollCount), std::enable_if_t<Empty, bool> = true>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived&, Visitor&) {}
+
+  // scalar initialization
+  template <int K = 0, bool Empty = (K == UnrollCount), bool Initialize = (K == 0), bool DoVectorOp = CanVectorize(K),
+            std::enable_if_t<!Empty && Initialize && !DoVectorOp, bool> = true>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    visitor.init(mat.coeff(0), 0);
+    run<1>(mat, visitor);
+  }
+
+  // scalar iteration
+  template <int K = 0, bool Empty = (K == UnrollCount), bool Initialize = (K == 0), bool DoVectorOp = CanVectorize(K),
+            std::enable_if_t<!Empty && !Initialize && !DoVectorOp, bool> = true>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    visitor(mat.coeff(K), K);
+    run<K + 1>(mat, visitor);
+  }
+
+  // vector initialization
+  template <int K = 0, bool Empty = (K == UnrollCount), bool Initialize = (K == 0), bool DoVectorOp = CanVectorize(K),
+            std::enable_if_t<!Empty && Initialize && DoVectorOp, bool> = true>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    Packet P = mat.template packet<Packet>(0);
+    visitor.initpacket(P, 0);
+    run<PacketSize>(mat, visitor);
+  }
+
+  // vector iteration
+  template <int K = 0, bool Empty = (K == UnrollCount), bool Initialize = (K == 0), bool DoVectorOp = CanVectorize(K),
+            std::enable_if_t<!Empty && !Initialize && DoVectorOp, bool> = true>
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    Packet P = mat.template packet<Packet>(K);
+    visitor.packet(P, K);
+    run<K + PacketSize>(mat, visitor);
+  }
+};
+
+// dynamic scalar outer-inner traversal
+template <typename Visitor, typename Derived, bool ShortCircuitEvaluation>
+struct visitor_impl<Visitor, Derived, Dynamic, /*Vectorize=*/false, /*LinearAccess=*/false, ShortCircuitEvaluation> {
+  using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
+  static constexpr bool RowMajor = Derived::IsRowMajor;
+
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    const Index innerSize = RowMajor ? mat.cols() : mat.rows();
+    const Index outerSize = RowMajor ? mat.rows() : mat.cols();
+    if (innerSize == 0 || outerSize == 0) return;
+    {
+      visitor.init(mat.coeff(0, 0), 0, 0);
+      if (short_circuit::run(visitor)) return;
+      for (Index i = 1; i < innerSize; ++i) {
+        Index r = RowMajor ? 0 : i;
+        Index c = RowMajor ? i : 0;
+        visitor(mat.coeff(r, c), r, c);
+        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+      }
+    }
+    for (Index j = 1; j < outerSize; j++) {
+      for (Index i = 0; i < innerSize; ++i) {
+        Index r = RowMajor ? j : i;
+        Index c = RowMajor ? i : j;
+        visitor(mat.coeff(r, c), r, c);
+        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+      }
+    }
+  }
+};
+
+// dynamic vectorized outer-inner traversal
+template <typename Visitor, typename Derived, bool ShortCircuitEvaluation>
+struct visitor_impl<Visitor, Derived, Dynamic, /*Vectorize=*/true, /*LinearAccess=*/false, ShortCircuitEvaluation> {
+  using Scalar = typename Derived::Scalar;
+  using Packet = typename packet_traits<Scalar>::type;
+  static constexpr int PacketSize = packet_traits<Scalar>::size;
+  using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
+  static constexpr bool RowMajor = Derived::IsRowMajor;
+
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    const Index innerSize = RowMajor ? mat.cols() : mat.rows();
+    const Index outerSize = RowMajor ? mat.rows() : mat.cols();
+    if (innerSize == 0 || outerSize == 0) return;
+    {
+      Index i = 0;
+      if (innerSize < PacketSize) {
+        visitor.init(mat.coeff(0, 0), 0, 0);
+        i = 1;
+      } else {
+        Packet p = mat.template packet<Packet>(0, 0);
+        visitor.initpacket(p, 0, 0);
+        i = PacketSize;
+      }
+      if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+      for (; i + PacketSize - 1 < innerSize; i += PacketSize) {
+        Index r = RowMajor ? 0 : i;
+        Index c = RowMajor ? i : 0;
+        Packet p = mat.template packet<Packet>(r, c);
+        visitor.packet(p, r, c);
+        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+      }
+      for (; i < innerSize; ++i) {
+        Index r = RowMajor ? 0 : i;
+        Index c = RowMajor ? i : 0;
+        visitor(mat.coeff(r, c), r, c);
+        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+      }
+    }
+    for (Index j = 1; j < outerSize; j++) {
+      Index i = 0;
+      for (; i + PacketSize - 1 < innerSize; i += PacketSize) {
+        Index r = RowMajor ? j : i;
+        Index c = RowMajor ? i : j;
+        Packet p = mat.template packet<Packet>(r, c);
+        visitor.packet(p, r, c);
+        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+      }
+      for (; i < innerSize; ++i) {
+        Index r = RowMajor ? j : i;
+        Index c = RowMajor ? i : j;
+        visitor(mat.coeff(r, c), r, c);
+        if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+      }
+    }
+  }
+};
+
+// dynamic scalar linear traversal
+template <typename Visitor, typename Derived, bool ShortCircuitEvaluation>
+struct visitor_impl<Visitor, Derived, Dynamic, /*Vectorize=*/false, /*LinearAccess=*/true, ShortCircuitEvaluation> {
+  using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
+
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    const Index size = mat.size();
+    if (size == 0) return;
+    visitor.init(mat.coeff(0), 0);
+    if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+    for (Index k = 1; k < size; k++) {
+      visitor(mat.coeff(k), k);
+      if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+    }
+  }
+};
+
+// dynamic vectorized linear traversal
+template <typename Visitor, typename Derived, bool ShortCircuitEvaluation>
+struct visitor_impl<Visitor, Derived, Dynamic, /*Vectorize=*/true, /*LinearAccess=*/true, ShortCircuitEvaluation> {
+  using Scalar = typename Derived::Scalar;
+  using Packet = typename packet_traits<Scalar>::type;
+  static constexpr int PacketSize = packet_traits<Scalar>::size;
+  using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>;
+
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) {
+    const Index size = mat.size();
+    if (size == 0) return;
+    Index k = 0;
+    if (size < PacketSize) {
+      visitor.init(mat.coeff(0), 0);
+      k = 1;
+    } else {
+      Packet p = mat.template packet<Packet>(k);
+      visitor.initpacket(p, k);
+      k = PacketSize;
+    }
+    if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+    for (; k + PacketSize - 1 < size; k += PacketSize) {
+      Packet p = mat.template packet<Packet>(k);
+      visitor.packet(p, k);
+      if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+    }
+    for (; k < size; k++) {
+      visitor(mat.coeff(k), k);
+      if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return;
+    }
+  }
+};
+
+// evaluator adaptor
+template <typename XprType>
+class visitor_evaluator {
+ public:
+  typedef evaluator<XprType> Evaluator;
+  typedef typename XprType::Scalar Scalar;
+  using Packet = typename packet_traits<Scalar>::type;
+  typedef std::remove_const_t<typename XprType::CoeffReturnType> CoeffReturnType;
+
+  static constexpr bool PacketAccess = static_cast<bool>(Evaluator::Flags & PacketAccessBit);
+  static constexpr bool LinearAccess = static_cast<bool>(Evaluator::Flags & LinearAccessBit);
+  static constexpr bool IsRowMajor = static_cast<bool>(XprType::IsRowMajor);
+  static constexpr int RowsAtCompileTime = XprType::RowsAtCompileTime;
+  static constexpr int ColsAtCompileTime = XprType::ColsAtCompileTime;
+  static constexpr int XprAlignment = Evaluator::Alignment;
+  static constexpr int CoeffReadCost = Evaluator::CoeffReadCost;
+
+  EIGEN_DEVICE_FUNC explicit visitor_evaluator(const XprType& xpr) : m_evaluator(xpr), m_xpr(xpr) {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_xpr.rows(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_xpr.cols(); }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index size() const EIGEN_NOEXCEPT { return m_xpr.size(); }
+  // outer-inner access
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
+    return m_evaluator.coeff(row, col);
+  }
+  template <typename Packet, int Alignment = Unaligned>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(Index row, Index col) const {
+    return m_evaluator.template packet<Alignment, Packet>(row, col);
+  }
+  // linear access
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_evaluator.coeff(index); }
+  template <typename Packet, int Alignment = XprAlignment>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(Index index) const {
+    return m_evaluator.template packet<Alignment, Packet>(index);
+  }
+
+ protected:
+  Evaluator m_evaluator;
+  const XprType& m_xpr;
+};
+
+template <typename Derived, typename Visitor, bool ShortCircuitEvaulation>
+struct visit_impl {
+  using Evaluator = visitor_evaluator<Derived>;
+  using Scalar = typename DenseBase<Derived>::Scalar;
+
+  static constexpr bool IsRowMajor = DenseBase<Derived>::IsRowMajor;
+  static constexpr int SizeAtCompileTime = DenseBase<Derived>::SizeAtCompileTime;
+  static constexpr int RowsAtCompileTime = DenseBase<Derived>::RowsAtCompileTime;
+  static constexpr int ColsAtCompileTime = DenseBase<Derived>::ColsAtCompileTime;
+  static constexpr int InnerSizeAtCompileTime = IsRowMajor ? ColsAtCompileTime : RowsAtCompileTime;
+  static constexpr int OuterSizeAtCompileTime = IsRowMajor ? RowsAtCompileTime : ColsAtCompileTime;
+
+  static constexpr bool LinearAccess =
+      Evaluator::LinearAccess && static_cast<bool>(functor_traits<Visitor>::LinearAccess);
+  static constexpr bool Vectorize = Evaluator::PacketAccess && static_cast<bool>(functor_traits<Visitor>::PacketAccess);
+
+  static constexpr int PacketSize = packet_traits<Scalar>::size;
+  static constexpr int VectorOps =
+      Vectorize ? (LinearAccess ? (SizeAtCompileTime / PacketSize)
+                                : (OuterSizeAtCompileTime * (InnerSizeAtCompileTime / PacketSize)))
+                : 0;
+  static constexpr int ScalarOps = SizeAtCompileTime - (VectorOps * PacketSize);
+  // treat vector op and scalar op as same cost for unroll logic
+  static constexpr int TotalOps = VectorOps + ScalarOps;
+
+  static constexpr int UnrollCost = int(Evaluator::CoeffReadCost) + int(functor_traits<Visitor>::Cost);
+  static constexpr bool Unroll = (SizeAtCompileTime != Dynamic) && ((TotalOps * UnrollCost) <= EIGEN_UNROLLING_LIMIT);
+  static constexpr int UnrollCount = Unroll ? int(SizeAtCompileTime) : Dynamic;
+
+  using impl = visitor_impl<Visitor, Evaluator, UnrollCount, Vectorize, LinearAccess, ShortCircuitEvaulation>;
+
+  static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const DenseBase<Derived>& mat, Visitor& visitor) {
+    Evaluator evaluator(mat.derived());
+    impl::run(evaluator, visitor);
+  }
+};
+
+}  // end namespace internal
+
+/** Applies the visitor \a visitor to the whole coefficients of the matrix or vector.
+ *
+ * The template parameter \a Visitor is the type of the visitor and provides the following interface:
+ * \code
+ * struct MyVisitor {
+ *   // called for the first coefficient
+ *   void init(const Scalar& value, Index i, Index j);
+ *   // called for all other coefficients
+ *   void operator() (const Scalar& value, Index i, Index j);
+ * };
+ * \endcode
+ *
+ * \note compared to one or two \em for \em loops, visitors offer automatic
+ * unrolling for small fixed size matrix.
+ *
+ * \note if the matrix is empty, then the visitor is left unchanged.
+ *
+ * \sa minCoeff(Index*,Index*), maxCoeff(Index*,Index*), DenseBase::redux()
+ */
+template <typename Derived>
+template <typename Visitor>
+EIGEN_DEVICE_FUNC void DenseBase<Derived>::visit(Visitor& visitor) const {
+  using impl = internal::visit_impl<Derived, Visitor, /*ShortCircuitEvaulation*/ false>;
+  impl::run(derived(), visitor);
+}
+
+namespace internal {
+
+/** \internal
+ * \brief Base class to implement min and max visitors
+ */
+template <typename Derived>
+struct coeff_visitor {
+  // default initialization to avoid countless invalid maybe-uninitialized warnings by gcc
+  EIGEN_DEVICE_FUNC coeff_visitor() : row(-1), col(-1), res(0) {}
+  typedef typename Derived::Scalar Scalar;
+  Index row, col;
+  Scalar res;
+  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index i, Index j) {
+    res = value;
+    row = i;
+    col = j;
+  }
+};
+
+template <typename Scalar, int NaNPropagation, bool is_min = true>
+struct minmax_compare {
+  typedef typename packet_traits<Scalar>::type Packet;
+  static EIGEN_DEVICE_FUNC inline bool compare(Scalar a, Scalar b) { return a < b; }
+  static EIGEN_DEVICE_FUNC inline Scalar predux(const Packet& p) { return predux_min<NaNPropagation>(p); }
+};
+
+template <typename Scalar, int NaNPropagation>
+struct minmax_compare<Scalar, NaNPropagation, false> {
+  typedef typename packet_traits<Scalar>::type Packet;
+  static EIGEN_DEVICE_FUNC inline bool compare(Scalar a, Scalar b) { return a > b; }
+  static EIGEN_DEVICE_FUNC inline Scalar predux(const Packet& p) { return predux_max<NaNPropagation>(p); }
+};
+
+// Default implementation used by non-floating types, where we do not
+// need special logic for NaN handling.
+template <typename Derived, bool is_min, int NaNPropagation,
+          bool isInt = NumTraits<typename Derived::Scalar>::IsInteger>
+struct minmax_coeff_visitor : coeff_visitor<Derived> {
+  using Scalar = typename Derived::Scalar;
+  using Packet = typename packet_traits<Scalar>::type;
+  using Comparator = minmax_compare<Scalar, NaNPropagation, is_min>;
+  static constexpr Index PacketSize = packet_traits<Scalar>::size;
+
+  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index i, Index j) {
+    if (Comparator::compare(value, this->res)) {
+      this->res = value;
+      this->row = i;
+      this->col = j;
+    }
+  }
+  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index i, Index j) {
+    Scalar value = Comparator::predux(p);
+    if (Comparator::compare(value, this->res)) {
+      const Packet range = preverse(plset<Packet>(Scalar(1)));
+      Packet mask = pcmp_eq(pset1<Packet>(value), p);
+      Index max_idx = PacketSize - static_cast<Index>(predux_max(pand(range, mask)));
+      this->res = value;
+      this->row = Derived::IsRowMajor ? i : i + max_idx;
+      this->col = Derived::IsRowMajor ? j + max_idx : j;
+    }
+  }
+  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index i, Index j) {
+    Scalar value = Comparator::predux(p);
+    const Packet range = preverse(plset<Packet>(Scalar(1)));
+    Packet mask = pcmp_eq(pset1<Packet>(value), p);
+    Index max_idx = PacketSize - static_cast<Index>(predux_max(pand(range, mask)));
+    this->res = value;
+    this->row = Derived::IsRowMajor ? i : i + max_idx;
+    this->col = Derived::IsRowMajor ? j + max_idx : j;
+  }
+};
+
+// Suppress NaN. The only case in which we return NaN is if the matrix is all NaN,
+// in which case, row=0, col=0 is returned for the location.
+template <typename Derived, bool is_min>
+struct minmax_coeff_visitor<Derived, is_min, PropagateNumbers, false> : coeff_visitor<Derived> {
+  typedef typename Derived::Scalar Scalar;
+  using Packet = typename packet_traits<Scalar>::type;
+  using Comparator = minmax_compare<Scalar, PropagateNumbers, is_min>;
+
+  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index i, Index j) {
+    if ((!(numext::isnan)(value) && (numext::isnan)(this->res)) || Comparator::compare(value, this->res)) {
+      this->res = value;
+      this->row = i;
+      this->col = j;
+    }
+  }
+  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index i, Index j) {
+    const Index PacketSize = packet_traits<Scalar>::size;
+    Scalar value = Comparator::predux(p);
+    if ((!(numext::isnan)(value) && (numext::isnan)(this->res)) || Comparator::compare(value, this->res)) {
+      const Packet range = preverse(plset<Packet>(Scalar(1)));
+      /* mask will be zero for NaNs, so they will be ignored. */
+      Packet mask = pcmp_eq(pset1<Packet>(value), p);
+      Index max_idx = PacketSize - static_cast<Index>(predux_max(pand(range, mask)));
+      this->res = value;
+      this->row = Derived::IsRowMajor ? i : i + max_idx;
+      this->col = Derived::IsRowMajor ? j + max_idx : j;
+    }
+  }
+  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index i, Index j) {
+    const Index PacketSize = packet_traits<Scalar>::size;
+    Scalar value = Comparator::predux(p);
+    if ((numext::isnan)(value)) {
+      this->res = value;
+      this->row = 0;
+      this->col = 0;
+      return;
+    }
+    const Packet range = preverse(plset<Packet>(Scalar(1)));
+    /* mask will be zero for NaNs, so they will be ignored. */
+    Packet mask = pcmp_eq(pset1<Packet>(value), p);
+    Index max_idx = PacketSize - static_cast<Index>(predux_max(pand(range, mask)));
+    this->res = value;
+    this->row = Derived::IsRowMajor ? i : i + max_idx;
+    this->col = Derived::IsRowMajor ? j + max_idx : j;
+  }
+};
+
+// Propagate NaNs. If the matrix contains NaN, the location of the first NaN
+// will be returned in row and col.
+template <typename Derived, bool is_min, int NaNPropagation>
+struct minmax_coeff_visitor<Derived, is_min, NaNPropagation, false> : coeff_visitor<Derived> {
+  typedef typename Derived::Scalar Scalar;
+  using Packet = typename packet_traits<Scalar>::type;
+  using Comparator = minmax_compare<Scalar, PropagateNaN, is_min>;
+
+  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index i, Index j) {
+    const bool value_is_nan = (numext::isnan)(value);
+    if ((value_is_nan && !(numext::isnan)(this->res)) || Comparator::compare(value, this->res)) {
+      this->res = value;
+      this->row = i;
+      this->col = j;
+    }
+  }
+  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index i, Index j) {
+    const Index PacketSize = packet_traits<Scalar>::size;
+    Scalar value = Comparator::predux(p);
+    const bool value_is_nan = (numext::isnan)(value);
+    if ((value_is_nan && !(numext::isnan)(this->res)) || Comparator::compare(value, this->res)) {
+      const Packet range = preverse(plset<Packet>(Scalar(1)));
+      // If the value is NaN, pick the first position of a NaN, otherwise pick the first extremal value.
+      Packet mask = value_is_nan ? pnot(pcmp_eq(p, p)) : pcmp_eq(pset1<Packet>(value), p);
+      Index max_idx = PacketSize - static_cast<Index>(predux_max(pand(range, mask)));
+      this->res = value;
+      this->row = Derived::IsRowMajor ? i : i + max_idx;
+      this->col = Derived::IsRowMajor ? j + max_idx : j;
+    }
+  }
+  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index i, Index j) {
+    const Index PacketSize = packet_traits<Scalar>::size;
+    Scalar value = Comparator::predux(p);
+    const bool value_is_nan = (numext::isnan)(value);
+    const Packet range = preverse(plset<Packet>(Scalar(1)));
+    // If the value is NaN, pick the first position of a NaN, otherwise pick the first extremal value.
+    Packet mask = value_is_nan ? pnot(pcmp_eq(p, p)) : pcmp_eq(pset1<Packet>(value), p);
+    Index max_idx = PacketSize - static_cast<Index>(predux_max(pand(range, mask)));
+    this->res = value;
+    this->row = Derived::IsRowMajor ? i : i + max_idx;
+    this->col = Derived::IsRowMajor ? j + max_idx : j;
+  }
+};
+
+template <typename Derived, bool is_min, int NaNPropagation>
+struct functor_traits<minmax_coeff_visitor<Derived, is_min, NaNPropagation>> {
+  using Scalar = typename Derived::Scalar;
+  enum { Cost = NumTraits<Scalar>::AddCost, LinearAccess = false, PacketAccess = packet_traits<Scalar>::HasCmp };
+};
+
+template <typename Scalar>
+struct all_visitor {
+  using result_type = bool;
+  using Packet = typename packet_traits<Scalar>::type;
+  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = (value != Scalar(0)); }
+  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = (value != Scalar(0)); }
+  EIGEN_DEVICE_FUNC inline bool all_predux(const Packet& p) const { return !predux_any(pcmp_eq(p, pzero(p))); }
+  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = all_predux(p); }
+  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = all_predux(p); }
+  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) { res = res && (value != Scalar(0)); }
+  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) { res = res && (value != Scalar(0)); }
+  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res = res && all_predux(p); }
+  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res = res && all_predux(p); }
+  EIGEN_DEVICE_FUNC inline bool done() const { return !res; }
+  bool res = true;
+};
+template <typename Scalar>
+struct functor_traits<all_visitor<Scalar>> {
+  enum { Cost = NumTraits<Scalar>::ReadCost, LinearAccess = true, PacketAccess = packet_traits<Scalar>::HasCmp };
+};
+
+template <typename Scalar>
+struct any_visitor {
+  using result_type = bool;
+  using Packet = typename packet_traits<Scalar>::type;
+  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = (value != Scalar(0)); }
+  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = (value != Scalar(0)); }
+  EIGEN_DEVICE_FUNC inline bool any_predux(const Packet& p) const {
+    return predux_any(pandnot(ptrue(p), pcmp_eq(p, pzero(p))));
+  }
+  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = any_predux(p); }
+  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = any_predux(p); }
+  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) { res = res || (value != Scalar(0)); }
+  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) { res = res || (value != Scalar(0)); }
+  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res = res || any_predux(p); }
+  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res = res || any_predux(p); }
+  EIGEN_DEVICE_FUNC inline bool done() const { return res; }
+  bool res = false;
+};
+template <typename Scalar>
+struct functor_traits<any_visitor<Scalar>> {
+  enum { Cost = NumTraits<Scalar>::ReadCost, LinearAccess = true, PacketAccess = packet_traits<Scalar>::HasCmp };
+};
+
+template <typename Scalar>
+struct count_visitor {
+  using result_type = Index;
+  using Packet = typename packet_traits<Scalar>::type;
+  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = value != Scalar(0) ? 1 : 0; }
+  EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = value != Scalar(0) ? 1 : 0; }
+  EIGEN_DEVICE_FUNC inline Index count_redux(const Packet& p) const {
+    const Packet cst_one = pset1<Packet>(Scalar(1));
+    Packet true_vals = pandnot(cst_one, pcmp_eq(p, pzero(p)));
+    Scalar num_true = predux(true_vals);
+    return static_cast<Index>(num_true);
+  }
+  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = count_redux(p); }
+  EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = count_redux(p); }
+  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) {
+    if (value != Scalar(0)) res++;
+  }
+  EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) {
+    if (value != Scalar(0)) res++;
+  }
+  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res += count_redux(p); }
+  EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res += count_redux(p); }
+  Index res = 0;
+};
+
+template <typename Scalar>
+struct functor_traits<count_visitor<Scalar>> {
+  enum {
+    Cost = NumTraits<Scalar>::AddCost,
+    LinearAccess = true,
+    // predux is problematic for bool
+    PacketAccess = packet_traits<Scalar>::HasCmp && packet_traits<Scalar>::HasAdd && !is_same<Scalar, bool>::value
+  };
+};
+
+}  // end namespace internal
+
+/** \fn DenseBase<Derived>::minCoeff(IndexType* rowId, IndexType* colId) const
+ * \returns the minimum of all coefficients of *this and puts in *row and *col its location.
+ *
+ * In case \c *this contains NaN, NaNPropagation determines the behavior:
+ *   NaNPropagation == PropagateFast : undefined
+ *   NaNPropagation == PropagateNaN : result is NaN
+ *   NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN
+ * \warning the matrix must be not empty, otherwise an assertion is triggered.
+ *
+ * \sa DenseBase::minCoeff(Index*), DenseBase::maxCoeff(Index*,Index*), DenseBase::visit(), DenseBase::minCoeff()
+ */
+template <typename Derived>
+template <int NaNPropagation, typename IndexType>
+EIGEN_DEVICE_FUNC typename internal::traits<Derived>::Scalar DenseBase<Derived>::minCoeff(IndexType* rowId,
+                                                                                          IndexType* colId) const {
+  eigen_assert(this->rows() > 0 && this->cols() > 0 && "you are using an empty matrix");
+
+  internal::minmax_coeff_visitor<Derived, true, NaNPropagation> minVisitor;
+  this->visit(minVisitor);
+  *rowId = minVisitor.row;
+  if (colId) *colId = minVisitor.col;
+  return minVisitor.res;
+}
+
+/** \returns the minimum of all coefficients of *this and puts in *index its location.
+ *
+ * In case \c *this contains NaN, NaNPropagation determines the behavior:
+ *   NaNPropagation == PropagateFast : undefined
+ *   NaNPropagation == PropagateNaN : result is NaN
+ *   NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN
+ * \warning the matrix must be not empty, otherwise an assertion is triggered.
+ *
+ * \sa DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::maxCoeff(IndexType*,IndexType*), DenseBase::visit(),
+ * DenseBase::minCoeff()
+ */
+template <typename Derived>
+template <int NaNPropagation, typename IndexType>
+EIGEN_DEVICE_FUNC typename internal::traits<Derived>::Scalar DenseBase<Derived>::minCoeff(IndexType* index) const {
+  eigen_assert(this->rows() > 0 && this->cols() > 0 && "you are using an empty matrix");
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+
+  internal::minmax_coeff_visitor<Derived, true, NaNPropagation> minVisitor;
+  this->visit(minVisitor);
+  *index = IndexType((RowsAtCompileTime == 1) ? minVisitor.col : minVisitor.row);
+  return minVisitor.res;
+}
+
+/** \fn DenseBase<Derived>::maxCoeff(IndexType* rowId, IndexType* colId) const
+ * \returns the maximum of all coefficients of *this and puts in *row and *col its location.
+ *
+ * In case \c *this contains NaN, NaNPropagation determines the behavior:
+ *   NaNPropagation == PropagateFast : undefined
+ *   NaNPropagation == PropagateNaN : result is NaN
+ *   NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN
+ * \warning the matrix must be not empty, otherwise an assertion is triggered.
+ *
+ * \sa DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::visit(), DenseBase::maxCoeff()
+ */
+template <typename Derived>
+template <int NaNPropagation, typename IndexType>
+EIGEN_DEVICE_FUNC typename internal::traits<Derived>::Scalar DenseBase<Derived>::maxCoeff(IndexType* rowPtr,
+                                                                                          IndexType* colPtr) const {
+  eigen_assert(this->rows() > 0 && this->cols() > 0 && "you are using an empty matrix");
+
+  internal::minmax_coeff_visitor<Derived, false, NaNPropagation> maxVisitor;
+  this->visit(maxVisitor);
+  *rowPtr = maxVisitor.row;
+  if (colPtr) *colPtr = maxVisitor.col;
+  return maxVisitor.res;
+}
+
+/** \returns the maximum of all coefficients of *this and puts in *index its location.
+ *
+ * In case \c *this contains NaN, NaNPropagation determines the behavior:
+ *   NaNPropagation == PropagateFast : undefined
+ *   NaNPropagation == PropagateNaN : result is NaN
+ *   NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN
+ * \warning the matrix must be not empty, otherwise an assertion is triggered.
+ *
+ * \sa DenseBase::maxCoeff(IndexType*,IndexType*), DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::visitor(),
+ * DenseBase::maxCoeff()
+ */
+template <typename Derived>
+template <int NaNPropagation, typename IndexType>
+EIGEN_DEVICE_FUNC typename internal::traits<Derived>::Scalar DenseBase<Derived>::maxCoeff(IndexType* index) const {
+  eigen_assert(this->rows() > 0 && this->cols() > 0 && "you are using an empty matrix");
+
+  EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
+  internal::minmax_coeff_visitor<Derived, false, NaNPropagation> maxVisitor;
+  this->visit(maxVisitor);
+  *index = (RowsAtCompileTime == 1) ? maxVisitor.col : maxVisitor.row;
+  return maxVisitor.res;
+}
+
+/** \returns true if all coefficients are true
+ *
+ * Example: \include MatrixBase_all.cpp
+ * Output: \verbinclude MatrixBase_all.out
+ *
+ * \sa any(), Cwise::operator<()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::all() const {
+  using Visitor = internal::all_visitor<Scalar>;
+  using impl = internal::visit_impl<Derived, Visitor, /*ShortCircuitEvaulation*/ true>;
+  Visitor visitor;
+  impl::run(derived(), visitor);
+  return visitor.res;
+}
+
+/** \returns true if at least one coefficient is true
+ *
+ * \sa all()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::any() const {
+  using Visitor = internal::any_visitor<Scalar>;
+  using impl = internal::visit_impl<Derived, Visitor, /*ShortCircuitEvaulation*/ true>;
+  Visitor visitor;
+  impl::run(derived(), visitor);
+  return visitor.res;
+}
+
+/** \returns the number of coefficients which evaluate to true
+ *
+ * \sa all(), any()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC Index DenseBase<Derived>::count() const {
+  using Visitor = internal::count_visitor<Scalar>;
+  using impl = internal::visit_impl<Derived, Visitor, /*ShortCircuitEvaulation*/ false>;
+  Visitor visitor;
+  impl::run(derived(), visitor);
+  return visitor.res;
+}
+
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::hasNaN() const {
+  return derived().cwiseTypedNotEqual(derived()).any();
+}
+
+/** \returns true if \c *this contains only finite numbers, i.e., no NaN and no +/-INF values.
+ *
+ * \sa hasNaN()
+ */
+template <typename Derived>
+EIGEN_DEVICE_FUNC inline bool DenseBase<Derived>::allFinite() const {
+  return derived().array().isFiniteTyped().all();
+}
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_VISITOR_H

include/Eigen/src/Core/arch/AVX/Complex.h

@@ -0,0 +1,530 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2014 Benoit Steiner (benoit.steiner.goog@gmail.com)
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_COMPLEX_AVX_H
+#define EIGEN_COMPLEX_AVX_H
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+//---------- float ----------
+struct Packet4cf {
+  EIGEN_STRONG_INLINE Packet4cf() {}
+  EIGEN_STRONG_INLINE explicit Packet4cf(const __m256& a) : v(a) {}
+  __m256 v;
+};
+
+#ifndef EIGEN_VECTORIZE_AVX512
+template <>
+struct packet_traits<std::complex<float> > : default_packet_traits {
+  typedef Packet4cf type;
+  typedef Packet2cf half;
+  enum {
+    Vectorizable = 1,
+    AlignedOnScalar = 1,
+    size = 4,
+
+    HasAdd = 1,
+    HasSub = 1,
+    HasMul = 1,
+    HasDiv = 1,
+    HasNegate = 1,
+    HasSqrt = 1,
+    HasLog = 1,
+    HasExp = 1,
+    HasAbs = 0,
+    HasAbs2 = 0,
+    HasMin = 0,
+    HasMax = 0,
+    HasSetLinear = 0
+  };
+};
+#endif
+
+template <>
+struct unpacket_traits<Packet4cf> {
+  typedef std::complex<float> type;
+  typedef Packet2cf half;
+  typedef Packet8f as_real;
+  enum {
+    size = 4,
+    alignment = Aligned32,
+    vectorizable = true,
+    masked_load_available = false,
+    masked_store_available = false
+  };
+};
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf padd<Packet4cf>(const Packet4cf& a, const Packet4cf& b) {
+  return Packet4cf(_mm256_add_ps(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf psub<Packet4cf>(const Packet4cf& a, const Packet4cf& b) {
+  return Packet4cf(_mm256_sub_ps(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf pnegate(const Packet4cf& a) {
+  return Packet4cf(pnegate(a.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf pconj(const Packet4cf& a) {
+  const __m256 mask = _mm256_castsi256_ps(_mm256_setr_epi32(0x00000000, 0x80000000, 0x00000000, 0x80000000, 0x00000000,
+                                                            0x80000000, 0x00000000, 0x80000000));
+  return Packet4cf(_mm256_xor_ps(a.v, mask));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) {
+  __m256 tmp1 = _mm256_mul_ps(_mm256_movehdup_ps(a.v), _mm256_permute_ps(b.v, _MM_SHUFFLE(2, 3, 0, 1)));
+  __m256 tmp2 = _mm256_moveldup_ps(a.v);
+#ifdef EIGEN_VECTORIZE_FMA
+  __m256 result = _mm256_fmaddsub_ps(tmp2, b.v, tmp1);
+#else
+  __m256 result = _mm256_addsub_ps(_mm256_mul_ps(tmp2, b.v), tmp1);
+#endif
+  return Packet4cf(result);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf pcmp_eq(const Packet4cf& a, const Packet4cf& b) {
+  __m256 eq = _mm256_cmp_ps(a.v, b.v, _CMP_EQ_OQ);
+  return Packet4cf(_mm256_and_ps(eq, _mm256_permute_ps(eq, 0xb1)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf ptrue<Packet4cf>(const Packet4cf& a) {
+  return Packet4cf(ptrue(Packet8f(a.v)));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf pand<Packet4cf>(const Packet4cf& a, const Packet4cf& b) {
+  return Packet4cf(_mm256_and_ps(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf por<Packet4cf>(const Packet4cf& a, const Packet4cf& b) {
+  return Packet4cf(_mm256_or_ps(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf pxor<Packet4cf>(const Packet4cf& a, const Packet4cf& b) {
+  return Packet4cf(_mm256_xor_ps(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf pandnot<Packet4cf>(const Packet4cf& a, const Packet4cf& b) {
+  return Packet4cf(_mm256_andnot_ps(b.v, a.v));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf pload<Packet4cf>(const std::complex<float>* from) {
+  EIGEN_DEBUG_ALIGNED_LOAD return Packet4cf(_mm256_load_ps(&numext::real_ref(*from)));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf ploadu<Packet4cf>(const std::complex<float>* from) {
+  EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cf(_mm256_loadu_ps(&numext::real_ref(*from)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf pset1<Packet4cf>(const std::complex<float>& from) {
+  const float re = std::real(from);
+  const float im = std::imag(from);
+  return Packet4cf(_mm256_set_ps(im, re, im, re, im, re, im, re));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf ploaddup<Packet4cf>(const std::complex<float>* from) {
+  // FIXME The following might be optimized using _mm256_movedup_pd
+  Packet2cf a = ploaddup<Packet2cf>(from);
+  Packet2cf b = ploaddup<Packet2cf>(from + 1);
+  return Packet4cf(_mm256_insertf128_ps(_mm256_castps128_ps256(a.v), b.v, 1));
+}
+
+template <>
+EIGEN_STRONG_INLINE void pstore<std::complex<float> >(std::complex<float>* to, const Packet4cf& from) {
+  EIGEN_DEBUG_ALIGNED_STORE _mm256_store_ps(&numext::real_ref(*to), from.v);
+}
+template <>
+EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet4cf& from) {
+  EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_ps(&numext::real_ref(*to), from.v);
+}
+
+template <>
+EIGEN_DEVICE_FUNC inline Packet4cf pgather<std::complex<float>, Packet4cf>(const std::complex<float>* from,
+                                                                           Index stride) {
+  return Packet4cf(_mm256_set_ps(std::imag(from[3 * stride]), std::real(from[3 * stride]), std::imag(from[2 * stride]),
+                                 std::real(from[2 * stride]), std::imag(from[1 * stride]), std::real(from[1 * stride]),
+                                 std::imag(from[0 * stride]), std::real(from[0 * stride])));
+}
+
+template <>
+EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet4cf>(std::complex<float>* to, const Packet4cf& from,
+                                                                       Index stride) {
+  __m128 low = _mm256_extractf128_ps(from.v, 0);
+  to[stride * 0] =
+      std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 0)), _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1)));
+  to[stride * 1] =
+      std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 2)), _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3)));
+
+  __m128 high = _mm256_extractf128_ps(from.v, 1);
+  to[stride * 2] =
+      std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 0)), _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1)));
+  to[stride * 3] =
+      std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 2)), _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3)));
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet4cf>(const Packet4cf& a) {
+  return pfirst(Packet2cf(_mm256_castps256_ps128(a.v)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf preverse(const Packet4cf& a) {
+  __m128 low = _mm256_extractf128_ps(a.v, 0);
+  __m128 high = _mm256_extractf128_ps(a.v, 1);
+  __m128d lowd = _mm_castps_pd(low);
+  __m128d highd = _mm_castps_pd(high);
+  low = _mm_castpd_ps(_mm_shuffle_pd(lowd, lowd, 0x1));
+  high = _mm_castpd_ps(_mm_shuffle_pd(highd, highd, 0x1));
+  __m256 result = _mm256_setzero_ps();
+  result = _mm256_insertf128_ps(result, low, 1);
+  result = _mm256_insertf128_ps(result, high, 0);
+  return Packet4cf(result);
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<float> predux<Packet4cf>(const Packet4cf& a) {
+  return predux(padd(Packet2cf(_mm256_extractf128_ps(a.v, 0)), Packet2cf(_mm256_extractf128_ps(a.v, 1))));
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet4cf>(const Packet4cf& a) {
+  return predux_mul(pmul(Packet2cf(_mm256_extractf128_ps(a.v, 0)), Packet2cf(_mm256_extractf128_ps(a.v, 1))));
+}
+
+EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cf, Packet8f)
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf pdiv<Packet4cf>(const Packet4cf& a, const Packet4cf& b) {
+  return pdiv_complex(a, b);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf pcplxflip<Packet4cf>(const Packet4cf& x) {
+  return Packet4cf(_mm256_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0, 1)));
+}
+
+//---------- double ----------
+struct Packet2cd {
+  EIGEN_STRONG_INLINE Packet2cd() {}
+  EIGEN_STRONG_INLINE explicit Packet2cd(const __m256d& a) : v(a) {}
+  __m256d v;
+};
+
+#ifndef EIGEN_VECTORIZE_AVX512
+template <>
+struct packet_traits<std::complex<double> > : default_packet_traits {
+  typedef Packet2cd type;
+  typedef Packet1cd half;
+  enum {
+    Vectorizable = 1,
+    AlignedOnScalar = 0,
+    size = 2,
+
+    HasAdd = 1,
+    HasSub = 1,
+    HasMul = 1,
+    HasDiv = 1,
+    HasNegate = 1,
+    HasSqrt = 1,
+    HasLog = 1,
+    HasAbs = 0,
+    HasAbs2 = 0,
+    HasMin = 0,
+    HasMax = 0,
+    HasSetLinear = 0
+  };
+};
+#endif
+
+template <>
+struct unpacket_traits<Packet2cd> {
+  typedef std::complex<double> type;
+  typedef Packet1cd half;
+  typedef Packet4d as_real;
+  enum {
+    size = 2,
+    alignment = Aligned32,
+    vectorizable = true,
+    masked_load_available = false,
+    masked_store_available = false
+  };
+};
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd padd<Packet2cd>(const Packet2cd& a, const Packet2cd& b) {
+  return Packet2cd(_mm256_add_pd(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd psub<Packet2cd>(const Packet2cd& a, const Packet2cd& b) {
+  return Packet2cd(_mm256_sub_pd(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd pnegate(const Packet2cd& a) {
+  return Packet2cd(pnegate(a.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd pconj(const Packet2cd& a) {
+  const __m256d mask = _mm256_castsi256_pd(_mm256_set_epi32(0x80000000, 0x0, 0x0, 0x0, 0x80000000, 0x0, 0x0, 0x0));
+  return Packet2cd(_mm256_xor_pd(a.v, mask));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) {
+  __m256d tmp1 = _mm256_mul_pd(_mm256_permute_pd(a.v, 0xF), _mm256_permute_pd(b.v, 0x5));
+  __m256d tmp2 = _mm256_movedup_pd(a.v);
+#ifdef EIGEN_VECTORIZE_FMA
+  __m256d result = _mm256_fmaddsub_pd(tmp2, b.v, tmp1);
+#else
+  __m256d result = _mm256_addsub_pd(_mm256_mul_pd(tmp2, b.v), tmp1);
+#endif
+  return Packet2cd(result);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd pcmp_eq(const Packet2cd& a, const Packet2cd& b) {
+  __m256d eq = _mm256_cmp_pd(a.v, b.v, _CMP_EQ_OQ);
+  return Packet2cd(pand(eq, _mm256_permute_pd(eq, 0x5)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd ptrue<Packet2cd>(const Packet2cd& a) {
+  return Packet2cd(ptrue(Packet4d(a.v)));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd pand<Packet2cd>(const Packet2cd& a, const Packet2cd& b) {
+  return Packet2cd(_mm256_and_pd(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd por<Packet2cd>(const Packet2cd& a, const Packet2cd& b) {
+  return Packet2cd(_mm256_or_pd(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd pxor<Packet2cd>(const Packet2cd& a, const Packet2cd& b) {
+  return Packet2cd(_mm256_xor_pd(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd pandnot<Packet2cd>(const Packet2cd& a, const Packet2cd& b) {
+  return Packet2cd(_mm256_andnot_pd(b.v, a.v));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd pload<Packet2cd>(const std::complex<double>* from) {
+  EIGEN_DEBUG_ALIGNED_LOAD return Packet2cd(_mm256_load_pd((const double*)from));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd ploadu<Packet2cd>(const std::complex<double>* from) {
+  EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cd(_mm256_loadu_pd((const double*)from));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd pset1<Packet2cd>(const std::complex<double>& from) {
+  // in case casting to a __m128d* is really not safe, then we can still fallback to this version: (much slower though)
+  //   return Packet2cd(_mm256_loadu2_m128d((const double*)&from,(const double*)&from));
+  return Packet2cd(_mm256_broadcast_pd((const __m128d*)(const void*)&from));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd ploaddup<Packet2cd>(const std::complex<double>* from) {
+  return pset1<Packet2cd>(*from);
+}
+
+template <>
+EIGEN_STRONG_INLINE void pstore<std::complex<double> >(std::complex<double>* to, const Packet2cd& from) {
+  EIGEN_DEBUG_ALIGNED_STORE _mm256_store_pd((double*)to, from.v);
+}
+template <>
+EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double>* to, const Packet2cd& from) {
+  EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_pd((double*)to, from.v);
+}
+
+template <>
+EIGEN_DEVICE_FUNC inline Packet2cd pgather<std::complex<double>, Packet2cd>(const std::complex<double>* from,
+                                                                            Index stride) {
+  return Packet2cd(_mm256_set_pd(std::imag(from[1 * stride]), std::real(from[1 * stride]), std::imag(from[0 * stride]),
+                                 std::real(from[0 * stride])));
+}
+
+template <>
+EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet2cd>(std::complex<double>* to, const Packet2cd& from,
+                                                                        Index stride) {
+  __m128d low = _mm256_extractf128_pd(from.v, 0);
+  to[stride * 0] = std::complex<double>(_mm_cvtsd_f64(low), _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1)));
+  __m128d high = _mm256_extractf128_pd(from.v, 1);
+  to[stride * 1] = std::complex<double>(_mm_cvtsd_f64(high), _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1)));
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet2cd>(const Packet2cd& a) {
+  __m128d low = _mm256_extractf128_pd(a.v, 0);
+  EIGEN_ALIGN16 double res[2];
+  _mm_store_pd(res, low);
+  return std::complex<double>(res[0], res[1]);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd preverse(const Packet2cd& a) {
+  __m256d result = _mm256_permute2f128_pd(a.v, a.v, 1);
+  return Packet2cd(result);
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<double> predux<Packet2cd>(const Packet2cd& a) {
+  return predux(padd(Packet1cd(_mm256_extractf128_pd(a.v, 0)), Packet1cd(_mm256_extractf128_pd(a.v, 1))));
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet2cd>(const Packet2cd& a) {
+  return predux(pmul(Packet1cd(_mm256_extractf128_pd(a.v, 0)), Packet1cd(_mm256_extractf128_pd(a.v, 1))));
+}
+
+EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cd, Packet4d)
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd pdiv<Packet2cd>(const Packet2cd& a, const Packet2cd& b) {
+  return pdiv_complex(a, b);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd pcplxflip<Packet2cd>(const Packet2cd& x) {
+  return Packet2cd(_mm256_shuffle_pd(x.v, x.v, 0x5));
+}
+
+EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4cf, 4>& kernel) {
+  __m256d P0 = _mm256_castps_pd(kernel.packet[0].v);
+  __m256d P1 = _mm256_castps_pd(kernel.packet[1].v);
+  __m256d P2 = _mm256_castps_pd(kernel.packet[2].v);
+  __m256d P3 = _mm256_castps_pd(kernel.packet[3].v);
+
+  __m256d T0 = _mm256_shuffle_pd(P0, P1, 15);
+  __m256d T1 = _mm256_shuffle_pd(P0, P1, 0);
+  __m256d T2 = _mm256_shuffle_pd(P2, P3, 15);
+  __m256d T3 = _mm256_shuffle_pd(P2, P3, 0);
+
+  kernel.packet[1].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 32));
+  kernel.packet[3].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 49));
+  kernel.packet[0].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 32));
+  kernel.packet[2].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 49));
+}
+
+EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet2cd, 2>& kernel) {
+  __m256d tmp = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 0 + (2 << 4));
+  kernel.packet[1].v = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 1 + (3 << 4));
+  kernel.packet[0].v = tmp;
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd psqrt<Packet2cd>(const Packet2cd& a) {
+  return psqrt_complex<Packet2cd>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf psqrt<Packet4cf>(const Packet4cf& a) {
+  return psqrt_complex<Packet4cf>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cd plog<Packet2cd>(const Packet2cd& a) {
+  return plog_complex<Packet2cd>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf plog<Packet4cf>(const Packet4cf& a) {
+  return plog_complex<Packet4cf>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf pexp<Packet4cf>(const Packet4cf& a) {
+  return pexp_complex<Packet4cf>(a);
+}
+
+#ifdef EIGEN_VECTORIZE_FMA
+// std::complex<float>
+template <>
+EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& a, const Packet4cf& b, const Packet4cf& c) {
+  __m256 a_odd = _mm256_movehdup_ps(a.v);
+  __m256 a_even = _mm256_moveldup_ps(a.v);
+  __m256 b_swap = _mm256_permute_ps(b.v, _MM_SHUFFLE(2, 3, 0, 1));
+  __m256 result = _mm256_fmaddsub_ps(a_even, b.v, _mm256_fmaddsub_ps(a_odd, b_swap, c.v));
+  return Packet4cf(result);
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf pmsub(const Packet4cf& a, const Packet4cf& b, const Packet4cf& c) {
+  __m256 a_odd = _mm256_movehdup_ps(a.v);
+  __m256 a_even = _mm256_moveldup_ps(a.v);
+  __m256 b_swap = _mm256_permute_ps(b.v, _MM_SHUFFLE(2, 3, 0, 1));
+  __m256 result = _mm256_fmaddsub_ps(a_even, b.v, _mm256_fmsubadd_ps(a_odd, b_swap, c.v));
+  return Packet4cf(result);
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf pnmadd(const Packet4cf& a, const Packet4cf& b, const Packet4cf& c) {
+  __m256 a_odd = _mm256_movehdup_ps(a.v);
+  __m256 a_even = _mm256_moveldup_ps(a.v);
+  __m256 b_swap = _mm256_permute_ps(b.v, _MM_SHUFFLE(2, 3, 0, 1));
+  __m256 result = _mm256_fmaddsub_ps(a_odd, b_swap, _mm256_fmaddsub_ps(a_even, b.v, c.v));
+  return Packet4cf(result);
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cf pnmsub(const Packet4cf& a, const Packet4cf& b, const Packet4cf& c) {
+  __m256 a_odd = _mm256_movehdup_ps(a.v);
+  __m256 a_even = _mm256_moveldup_ps(a.v);
+  __m256 b_swap = _mm256_permute_ps(b.v, _MM_SHUFFLE(2, 3, 0, 1));
+  __m256 result = _mm256_fmaddsub_ps(a_odd, b_swap, _mm256_fmsubadd_ps(a_even, b.v, c.v));
+  return Packet4cf(result);
+}
+// std::complex<double>
+template <>
+EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& a, const Packet2cd& b, const Packet2cd& c) {
+  __m256d a_odd = _mm256_permute_pd(a.v, 0xF);
+  __m256d a_even = _mm256_movedup_pd(a.v);
+  __m256d b_swap = _mm256_permute_pd(b.v, 0x5);
+  __m256d result = _mm256_fmaddsub_pd(a_even, b.v, _mm256_fmaddsub_pd(a_odd, b_swap, c.v));
+  return Packet2cd(result);
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd pmsub(const Packet2cd& a, const Packet2cd& b, const Packet2cd& c) {
+  __m256d a_odd = _mm256_permute_pd(a.v, 0xF);
+  __m256d a_even = _mm256_movedup_pd(a.v);
+  __m256d b_swap = _mm256_permute_pd(b.v, 0x5);
+  __m256d result = _mm256_fmaddsub_pd(a_even, b.v, _mm256_fmsubadd_pd(a_odd, b_swap, c.v));
+  return Packet2cd(result);
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd pnmadd(const Packet2cd& a, const Packet2cd& b, const Packet2cd& c) {
+  __m256d a_odd = _mm256_permute_pd(a.v, 0xF);
+  __m256d a_even = _mm256_movedup_pd(a.v);
+  __m256d b_swap = _mm256_permute_pd(b.v, 0x5);
+  __m256d result = _mm256_fmaddsub_pd(a_odd, b_swap, _mm256_fmaddsub_pd(a_even, b.v, c.v));
+  return Packet2cd(result);
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cd pnmsub(const Packet2cd& a, const Packet2cd& b, const Packet2cd& c) {
+  __m256d a_odd = _mm256_permute_pd(a.v, 0xF);
+  __m256d a_even = _mm256_movedup_pd(a.v);
+  __m256d b_swap = _mm256_permute_pd(b.v, 0x5);
+  __m256d result = _mm256_fmaddsub_pd(a_odd, b_swap, _mm256_fmsubadd_pd(a_even, b.v, c.v));
+  return Packet2cd(result);
+}
+#endif
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_COMPLEX_AVX_H

include/Eigen/src/Core/arch/AVX/MathFunctions.h

@@ -0,0 +1,126 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com)
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_MATH_FUNCTIONS_AVX_H
+#define EIGEN_MATH_FUNCTIONS_AVX_H
+
+/* The sin and cos functions of this file are loosely derived from
+ * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/
+ */
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet8f)
+
+EIGEN_DOUBLE_PACKET_FUNCTION(atanh, Packet4d)
+EIGEN_DOUBLE_PACKET_FUNCTION(log, Packet4d)
+EIGEN_DOUBLE_PACKET_FUNCTION(log2, Packet4d)
+EIGEN_DOUBLE_PACKET_FUNCTION(exp, Packet4d)
+EIGEN_DOUBLE_PACKET_FUNCTION(tanh, Packet4d)
+#ifdef EIGEN_VECTORIZE_AVX2
+EIGEN_DOUBLE_PACKET_FUNCTION(sin, Packet4d)
+EIGEN_DOUBLE_PACKET_FUNCTION(cos, Packet4d)
+#endif
+EIGEN_GENERIC_PACKET_FUNCTION(atan, Packet4d)
+EIGEN_GENERIC_PACKET_FUNCTION(exp2, Packet4d)
+
+// Notice that for newer processors, it is counterproductive to use Newton
+// iteration for square root. In particular, Skylake and Zen2 processors
+// have approximately doubled throughput of the _mm_sqrt_ps instruction
+// compared to their predecessors.
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8f psqrt<Packet8f>(const Packet8f& _x) {
+  return _mm256_sqrt_ps(_x);
+}
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4d psqrt<Packet4d>(const Packet4d& _x) {
+  return _mm256_sqrt_pd(_x);
+}
+
+// Even on Skylake, using Newton iteration is a win for reciprocal square root.
+#if EIGEN_FAST_MATH
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8f prsqrt<Packet8f>(const Packet8f& a) {
+  // _mm256_rsqrt_ps returns -inf for negative denormals.
+  // _mm512_rsqrt**_ps returns -NaN for negative denormals.  We may want
+  // consistency here.
+  // const Packet8f rsqrt = pselect(pcmp_lt(a, pzero(a)),
+  //                                pset1<Packet8f>(-NumTraits<float>::quiet_NaN()),
+  //                                _mm256_rsqrt_ps(a));
+  return generic_rsqrt_newton_step<Packet8f, /*Steps=*/1>::run(a, _mm256_rsqrt_ps(a));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8f preciprocal<Packet8f>(const Packet8f& a) {
+  return generic_reciprocal_newton_step<Packet8f, /*Steps=*/1>::run(a, _mm256_rcp_ps(a));
+}
+
+#endif
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pfrexp(const Packet8h& a, Packet8h& exponent) {
+  Packet8f fexponent;
+  const Packet8h out = float2half(pfrexp<Packet8f>(half2float(a), fexponent));
+  exponent = float2half(fexponent);
+  return out;
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pldexp(const Packet8h& a, const Packet8h& exponent) {
+  return float2half(pldexp<Packet8f>(half2float(a), half2float(exponent)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8bf pfrexp(const Packet8bf& a, Packet8bf& exponent) {
+  Packet8f fexponent;
+  const Packet8bf out = F32ToBf16(pfrexp<Packet8f>(Bf16ToF32(a), fexponent));
+  exponent = F32ToBf16(fexponent);
+  return out;
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8bf pldexp(const Packet8bf& a, const Packet8bf& exponent) {
+  return F32ToBf16(pldexp<Packet8f>(Bf16ToF32(a), Bf16ToF32(exponent)));
+}
+
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pcos)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pexp)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pexp2)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pexpm1)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog1p)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, plog2)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, preciprocal)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, prsqrt)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, psin)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, psqrt)
+BF16_PACKET_FUNCTION(Packet8f, Packet8bf, ptanh)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, pcos)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, pexp)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, pexp2)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, pexpm1)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, plog)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, plog1p)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, plog2)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, preciprocal)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, prsqrt)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, psin)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, psqrt)
+F16_PACKET_FUNCTION(Packet8f, Packet8h, ptanh)
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_MATH_FUNCTIONS_AVX_H

include/Eigen/src/Core/arch/AVX/TypeCasting.h

@@ -0,0 +1,306 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2015 Benoit Steiner <benoit.steiner.goog@gmail.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_TYPE_CASTING_AVX_H
+#define EIGEN_TYPE_CASTING_AVX_H
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+#ifndef EIGEN_VECTORIZE_AVX512
+template <>
+struct type_casting_traits<float, bool> : vectorized_type_casting_traits<float, bool> {};
+template <>
+struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {};
+
+template <>
+struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {};
+template <>
+struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {};
+
+template <>
+struct type_casting_traits<float, double> : vectorized_type_casting_traits<float, double> {};
+template <>
+struct type_casting_traits<double, float> : vectorized_type_casting_traits<double, float> {};
+
+template <>
+struct type_casting_traits<double, int> : vectorized_type_casting_traits<double, int> {};
+template <>
+struct type_casting_traits<int, double> : vectorized_type_casting_traits<int, double> {};
+
+template <>
+struct type_casting_traits<half, float> : vectorized_type_casting_traits<half, float> {};
+template <>
+struct type_casting_traits<float, half> : vectorized_type_casting_traits<float, half> {};
+
+template <>
+struct type_casting_traits<bfloat16, float> : vectorized_type_casting_traits<bfloat16, float> {};
+template <>
+struct type_casting_traits<float, bfloat16> : vectorized_type_casting_traits<float, bfloat16> {};
+
+#ifdef EIGEN_VECTORIZE_AVX2
+template <>
+struct type_casting_traits<double, int64_t> : vectorized_type_casting_traits<double, int64_t> {};
+template <>
+struct type_casting_traits<int64_t, double> : vectorized_type_casting_traits<int64_t, double> {};
+#endif
+#endif
+
+template <>
+EIGEN_STRONG_INLINE Packet16b pcast<Packet8f, Packet16b>(const Packet8f& a, const Packet8f& b) {
+  __m256 nonzero_a = _mm256_cmp_ps(a, pzero(a), _CMP_NEQ_UQ);
+  __m256 nonzero_b = _mm256_cmp_ps(b, pzero(b), _CMP_NEQ_UQ);
+  constexpr char kFF = '\255';
+#ifndef EIGEN_VECTORIZE_AVX2
+  __m128i shuffle_mask128_a_lo = _mm_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0);
+  __m128i shuffle_mask128_a_hi = _mm_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF);
+  __m128i shuffle_mask128_b_lo = _mm_set_epi8(kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF);
+  __m128i shuffle_mask128_b_hi = _mm_set_epi8(12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF);
+  __m128i a_hi = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_a), 1), shuffle_mask128_a_hi);
+  __m128i a_lo = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_a), 0), shuffle_mask128_a_lo);
+  __m128i b_hi = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_b), 1), shuffle_mask128_b_hi);
+  __m128i b_lo = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_b), 0), shuffle_mask128_b_lo);
+  __m128i merged = _mm_or_si128(_mm_or_si128(b_lo, b_hi), _mm_or_si128(a_lo, a_hi));
+  return _mm_and_si128(merged, _mm_set1_epi8(1));
+#else
+  __m256i a_shuffle_mask = _mm256_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF,
+                                           kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0);
+  __m256i b_shuffle_mask = _mm256_set_epi8(12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF,
+                                           kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF);
+  __m256i a_shuff = _mm256_shuffle_epi8(_mm256_castps_si256(nonzero_a), a_shuffle_mask);
+  __m256i b_shuff = _mm256_shuffle_epi8(_mm256_castps_si256(nonzero_b), b_shuffle_mask);
+  __m256i a_or_b = _mm256_or_si256(a_shuff, b_shuff);
+  __m256i merged = _mm256_or_si256(a_or_b, _mm256_castsi128_si256(_mm256_extractf128_si256(a_or_b, 1)));
+  return _mm256_castsi256_si128(_mm256_and_si256(merged, _mm256_set1_epi8(1)));
+#endif
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8f pcast<Packet16b, Packet8f>(const Packet16b& a) {
+  const __m256 cst_one = _mm256_set1_ps(1.0f);
+#ifdef EIGEN_VECTORIZE_AVX2
+  __m256i a_extended = _mm256_cvtepi8_epi32(a);
+  __m256i abcd_efgh = _mm256_cmpeq_epi32(a_extended, _mm256_setzero_si256());
+#else
+  __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128());
+  __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop);
+  __m128i aaaa_bbbb_cccc_dddd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
+  __m128i eeee_ffff_gggg_hhhh = _mm_unpackhi_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
+  __m256i abcd_efgh = _mm256_setr_m128i(aaaa_bbbb_cccc_dddd, eeee_ffff_gggg_hhhh);
+#endif
+  __m256 result = _mm256_andnot_ps(_mm256_castsi256_ps(abcd_efgh), cst_one);
+  return result;
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8i pcast<Packet8f, Packet8i>(const Packet8f& a) {
+  return _mm256_cvttps_epi32(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8i pcast<Packet4d, Packet8i>(const Packet4d& a, const Packet4d& b) {
+  return _mm256_set_m128i(_mm256_cvttpd_epi32(b), _mm256_cvttpd_epi32(a));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4i pcast<Packet4d, Packet4i>(const Packet4d& a) {
+  return _mm256_cvttpd_epi32(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8f pcast<Packet8i, Packet8f>(const Packet8i& a) {
+  return _mm256_cvtepi32_ps(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8f pcast<Packet4d, Packet8f>(const Packet4d& a, const Packet4d& b) {
+  return _mm256_set_m128(_mm256_cvtpd_ps(b), _mm256_cvtpd_ps(a));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4f pcast<Packet4d, Packet4f>(const Packet4d& a) {
+  return _mm256_cvtpd_ps(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4d pcast<Packet8i, Packet4d>(const Packet8i& a) {
+  return _mm256_cvtepi32_pd(_mm256_castsi256_si128(a));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4d pcast<Packet4i, Packet4d>(const Packet4i& a) {
+  return _mm256_cvtepi32_pd(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4d pcast<Packet8f, Packet4d>(const Packet8f& a) {
+  return _mm256_cvtps_pd(_mm256_castps256_ps128(a));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4d pcast<Packet4f, Packet4d>(const Packet4f& a) {
+  return _mm256_cvtps_pd(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8i preinterpret<Packet8i, Packet8f>(const Packet8f& a) {
+  return _mm256_castps_si256(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8f preinterpret<Packet8f, Packet8i>(const Packet8i& a) {
+  return _mm256_castsi256_ps(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8ui preinterpret<Packet8ui, Packet8i>(const Packet8i& a) {
+  return Packet8ui(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8i preinterpret<Packet8i, Packet8ui>(const Packet8ui& a) {
+  return Packet8i(a);
+}
+
+// truncation operations
+
+template <>
+EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet8f>(const Packet8f& a) {
+  return _mm256_castps256_ps128(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4d>(const Packet4d& a) {
+  return _mm256_castpd256_pd128(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet8i>(const Packet8i& a) {
+  return _mm256_castsi256_si128(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet8ui>(const Packet8ui& a) {
+  return _mm256_castsi256_si128(a);
+}
+
+#ifdef EIGEN_VECTORIZE_AVX2
+template <>
+EIGEN_STRONG_INLINE Packet4l pcast<Packet4d, Packet4l>(const Packet4d& a) {
+#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVS512VL)
+  return _mm256_cvttpd_epi64(a);
+#else
+
+  // if 'a' exceeds the numerical limits of int64_t, the behavior is undefined
+
+  // e <= 0 corresponds to |a| < 1, which should result in zero. incidentally, intel intrinsics with shift arguments
+  // greater than or equal to 64 produce zero. furthermore, negative shifts appear to be interpreted as large positive
+  // shifts (two's complement), which also result in zero. therefore, e does not need to be clamped to [0, 64)
+
+  constexpr int kTotalBits = sizeof(double) * CHAR_BIT, kMantissaBits = std::numeric_limits<double>::digits - 1,
+                kExponentBits = kTotalBits - kMantissaBits - 1, kBias = (1 << (kExponentBits - 1)) - 1;
+
+  const __m256i cst_one = _mm256_set1_epi64x(1);
+  const __m256i cst_total_bits = _mm256_set1_epi64x(kTotalBits);
+  const __m256i cst_bias = _mm256_set1_epi64x(kBias);
+
+  __m256i a_bits = _mm256_castpd_si256(a);
+  // shift left by 1 to clear the sign bit, and shift right by kMantissaBits + 1 to recover biased exponent
+  __m256i biased_e = _mm256_srli_epi64(_mm256_slli_epi64(a_bits, 1), kMantissaBits + 1);
+  __m256i e = _mm256_sub_epi64(biased_e, cst_bias);
+
+  // shift to the left by kExponentBits + 1 to clear the sign and exponent bits
+  __m256i shifted_mantissa = _mm256_slli_epi64(a_bits, kExponentBits + 1);
+  // shift to the right by kTotalBits - e to convert the significand to an integer
+  __m256i result_significand = _mm256_srlv_epi64(shifted_mantissa, _mm256_sub_epi64(cst_total_bits, e));
+
+  // add the implied bit
+  __m256i result_exponent = _mm256_sllv_epi64(cst_one, e);
+  // e <= 0 is interpreted as a large positive shift (2's complement), which also conveniently results in zero
+  __m256i result = _mm256_add_epi64(result_significand, result_exponent);
+  // handle negative arguments
+  __m256i sign_mask = _mm256_cmpgt_epi64(_mm256_setzero_si256(), a_bits);
+  result = _mm256_sub_epi64(_mm256_xor_si256(result, sign_mask), sign_mask);
+  return result;
+#endif
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4d pcast<Packet4l, Packet4d>(const Packet4l& a) {
+#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVS512VL)
+  return _mm256_cvtepi64_pd(a);
+#else
+  EIGEN_ALIGN16 int64_t aux[4];
+  pstore(aux, a);
+  return _mm256_set_pd(static_cast<double>(aux[3]), static_cast<double>(aux[2]), static_cast<double>(aux[1]),
+                       static_cast<double>(aux[0]));
+#endif
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4d pcast<Packet2l, Packet4d>(const Packet2l& a, const Packet2l& b) {
+  return _mm256_set_m128d((pcast<Packet2l, Packet2d>(b)), (pcast<Packet2l, Packet2d>(a)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4ul preinterpret<Packet4ul, Packet4l>(const Packet4l& a) {
+  return Packet4ul(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4l preinterpret<Packet4l, Packet4ul>(const Packet4ul& a) {
+  return Packet4l(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4l preinterpret<Packet4l, Packet4d>(const Packet4d& a) {
+  return _mm256_castpd_si256(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4d preinterpret<Packet4d, Packet4l>(const Packet4l& a) {
+  return _mm256_castsi256_pd(a);
+}
+
+// truncation operations
+template <>
+EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet4l>(const Packet4l& a) {
+  return _mm256_castsi256_si128(a);
+}
+#endif
+
+template <>
+EIGEN_STRONG_INLINE Packet8f pcast<Packet8h, Packet8f>(const Packet8h& a) {
+  return half2float(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8f pcast<Packet8bf, Packet8f>(const Packet8bf& a) {
+  return Bf16ToF32(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pcast<Packet8f, Packet8h>(const Packet8f& a) {
+  return float2half(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8bf pcast<Packet8f, Packet8bf>(const Packet8f& a) {
+  return F32ToBf16(a);
+}
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_TYPE_CASTING_AVX_H

include/Eigen/src/Core/arch/AVX512/Complex.h

@@ -0,0 +1,472 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2018 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_COMPLEX_AVX512_H
+#define EIGEN_COMPLEX_AVX512_H
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+//---------- float ----------
+struct Packet8cf {
+  EIGEN_STRONG_INLINE Packet8cf() {}
+  EIGEN_STRONG_INLINE explicit Packet8cf(const __m512& a) : v(a) {}
+  __m512 v;
+};
+
+template <>
+struct packet_traits<std::complex<float> > : default_packet_traits {
+  typedef Packet8cf type;
+  typedef Packet4cf half;
+  enum {
+    Vectorizable = 1,
+    AlignedOnScalar = 1,
+    size = 8,
+
+    HasAdd = 1,
+    HasSub = 1,
+    HasMul = 1,
+    HasDiv = 1,
+    HasNegate = 1,
+    HasSqrt = 1,
+    HasLog = 1,
+    HasExp = 1,
+    HasAbs = 0,
+    HasAbs2 = 0,
+    HasMin = 0,
+    HasMax = 0,
+    HasSetLinear = 0
+  };
+};
+
+template <>
+struct unpacket_traits<Packet8cf> {
+  typedef std::complex<float> type;
+  typedef Packet4cf half;
+  typedef Packet16f as_real;
+  enum {
+    size = 8,
+    alignment = unpacket_traits<Packet16f>::alignment,
+    vectorizable = true,
+    masked_load_available = false,
+    masked_store_available = false
+  };
+};
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf ptrue<Packet8cf>(const Packet8cf& a) {
+  return Packet8cf(ptrue(Packet16f(a.v)));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8cf padd<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
+  return Packet8cf(_mm512_add_ps(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8cf psub<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
+  return Packet8cf(_mm512_sub_ps(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8cf pnegate(const Packet8cf& a) {
+  return Packet8cf(pnegate(a.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8cf pconj(const Packet8cf& a) {
+  const __m512 mask = _mm512_castsi512_ps(_mm512_setr_epi32(
+      0x00000000, 0x80000000, 0x00000000, 0x80000000, 0x00000000, 0x80000000, 0x00000000, 0x80000000, 0x00000000,
+      0x80000000, 0x00000000, 0x80000000, 0x00000000, 0x80000000, 0x00000000, 0x80000000));
+  return Packet8cf(pxor(a.v, mask));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf pmul<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
+  __m512 tmp2 = _mm512_mul_ps(_mm512_movehdup_ps(a.v), _mm512_permute_ps(b.v, _MM_SHUFFLE(2, 3, 0, 1)));
+  return Packet8cf(_mm512_fmaddsub_ps(_mm512_moveldup_ps(a.v), b.v, tmp2));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf pand<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
+  return Packet8cf(pand(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8cf por<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
+  return Packet8cf(por(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8cf pxor<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
+  return Packet8cf(pxor(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8cf pandnot<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
+  return Packet8cf(pandnot(a.v, b.v));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf pcmp_eq(const Packet8cf& a, const Packet8cf& b) {
+  __m512 eq = pcmp_eq<Packet16f>(a.v, b.v);
+  return Packet8cf(pand(eq, _mm512_permute_ps(eq, 0xB1)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf pload<Packet8cf>(const std::complex<float>* from) {
+  EIGEN_DEBUG_ALIGNED_LOAD return Packet8cf(pload<Packet16f>(&numext::real_ref(*from)));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8cf ploadu<Packet8cf>(const std::complex<float>* from) {
+  EIGEN_DEBUG_UNALIGNED_LOAD return Packet8cf(ploadu<Packet16f>(&numext::real_ref(*from)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf pset1<Packet8cf>(const std::complex<float>& from) {
+  const float re = std::real(from);
+  const float im = std::imag(from);
+  return Packet8cf(_mm512_set_ps(im, re, im, re, im, re, im, re, im, re, im, re, im, re, im, re));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf ploaddup<Packet8cf>(const std::complex<float>* from) {
+  return Packet8cf(_mm512_castpd_ps(ploaddup<Packet8d>((const double*)(const void*)from)));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8cf ploadquad<Packet8cf>(const std::complex<float>* from) {
+  return Packet8cf(_mm512_castpd_ps(ploadquad<Packet8d>((const double*)(const void*)from)));
+}
+
+template <>
+EIGEN_STRONG_INLINE void pstore<std::complex<float> >(std::complex<float>* to, const Packet8cf& from) {
+  EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v);
+}
+template <>
+EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet8cf& from) {
+  EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v);
+}
+
+template <>
+EIGEN_DEVICE_FUNC inline Packet8cf pgather<std::complex<float>, Packet8cf>(const std::complex<float>* from,
+                                                                           Index stride) {
+  return Packet8cf(_mm512_castpd_ps(pgather<double, Packet8d>((const double*)(const void*)from, stride)));
+}
+
+template <>
+EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet8cf>(std::complex<float>* to, const Packet8cf& from,
+                                                                       Index stride) {
+  pscatter((double*)(void*)to, _mm512_castps_pd(from.v), stride);
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet8cf>(const Packet8cf& a) {
+  return pfirst(Packet2cf(_mm512_castps512_ps128(a.v)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf preverse(const Packet8cf& a) {
+  return Packet8cf(_mm512_castsi512_ps(_mm512_permutexvar_epi64(
+      _mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), _mm512_castps_si512(a.v))));
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<float> predux<Packet8cf>(const Packet8cf& a) {
+  return predux(padd(Packet4cf(extract256<0>(a.v)), Packet4cf(extract256<1>(a.v))));
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet8cf>(const Packet8cf& a) {
+  return predux_mul(pmul(Packet4cf(extract256<0>(a.v)), Packet4cf(extract256<1>(a.v))));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cf predux_half_dowto4<Packet8cf>(const Packet8cf& a) {
+  __m256 lane0 = extract256<0>(a.v);
+  __m256 lane1 = extract256<1>(a.v);
+  __m256 res = _mm256_add_ps(lane0, lane1);
+  return Packet4cf(res);
+}
+
+EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet8cf, Packet16f)
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf pdiv<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
+  return pdiv_complex(a, b);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf pcplxflip<Packet8cf>(const Packet8cf& x) {
+  return Packet8cf(_mm512_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0, 1)));
+}
+
+//---------- double ----------
+struct Packet4cd {
+  EIGEN_STRONG_INLINE Packet4cd() {}
+  EIGEN_STRONG_INLINE explicit Packet4cd(const __m512d& a) : v(a) {}
+  __m512d v;
+};
+
+template <>
+struct packet_traits<std::complex<double> > : default_packet_traits {
+  typedef Packet4cd type;
+  typedef Packet2cd half;
+  enum {
+    Vectorizable = 1,
+    AlignedOnScalar = 0,
+    size = 4,
+
+    HasAdd = 1,
+    HasSub = 1,
+    HasMul = 1,
+    HasDiv = 1,
+    HasNegate = 1,
+    HasSqrt = 1,
+    HasLog = 1,
+    HasAbs = 0,
+    HasAbs2 = 0,
+    HasMin = 0,
+    HasMax = 0,
+    HasSetLinear = 0
+  };
+};
+
+template <>
+struct unpacket_traits<Packet4cd> {
+  typedef std::complex<double> type;
+  typedef Packet2cd half;
+  typedef Packet8d as_real;
+  enum {
+    size = 4,
+    alignment = unpacket_traits<Packet8d>::alignment,
+    vectorizable = true,
+    masked_load_available = false,
+    masked_store_available = false
+  };
+};
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd padd<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
+  return Packet4cd(_mm512_add_pd(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cd psub<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
+  return Packet4cd(_mm512_sub_pd(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cd pnegate(const Packet4cd& a) {
+  return Packet4cd(pnegate(a.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cd pconj(const Packet4cd& a) {
+  const __m512d mask = _mm512_castsi512_pd(_mm512_set_epi32(0x80000000, 0x0, 0x0, 0x0, 0x80000000, 0x0, 0x0, 0x0,
+                                                            0x80000000, 0x0, 0x0, 0x0, 0x80000000, 0x0, 0x0, 0x0));
+  return Packet4cd(pxor(a.v, mask));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd pmul<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
+  __m512d tmp1 = _mm512_shuffle_pd(a.v, a.v, 0x0);
+  __m512d tmp2 = _mm512_shuffle_pd(a.v, a.v, 0xFF);
+  __m512d tmp3 = _mm512_shuffle_pd(b.v, b.v, 0x55);
+  __m512d odd = _mm512_mul_pd(tmp2, tmp3);
+  return Packet4cd(_mm512_fmaddsub_pd(tmp1, b.v, odd));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd ptrue<Packet4cd>(const Packet4cd& a) {
+  return Packet4cd(ptrue(Packet8d(a.v)));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cd pand<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
+  return Packet4cd(pand(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cd por<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
+  return Packet4cd(por(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cd pxor<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
+  return Packet4cd(pxor(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cd pandnot<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
+  return Packet4cd(pandnot(a.v, b.v));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd pcmp_eq(const Packet4cd& a, const Packet4cd& b) {
+  __m512d eq = pcmp_eq<Packet8d>(a.v, b.v);
+  return Packet4cd(pand(eq, _mm512_permute_pd(eq, 0x55)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd pload<Packet4cd>(const std::complex<double>* from) {
+  EIGEN_DEBUG_ALIGNED_LOAD return Packet4cd(pload<Packet8d>((const double*)from));
+}
+template <>
+EIGEN_STRONG_INLINE Packet4cd ploadu<Packet4cd>(const std::complex<double>* from) {
+  EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cd(ploadu<Packet8d>((const double*)from));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd pset1<Packet4cd>(const std::complex<double>& from) {
+  return Packet4cd(_mm512_castps_pd(_mm512_broadcast_f32x4(_mm_castpd_ps(pset1<Packet1cd>(from).v))));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd ploaddup<Packet4cd>(const std::complex<double>* from) {
+  return Packet4cd(
+      _mm512_insertf64x4(_mm512_castpd256_pd512(ploaddup<Packet2cd>(from).v), ploaddup<Packet2cd>(from + 1).v, 1));
+}
+
+template <>
+EIGEN_STRONG_INLINE void pstore<std::complex<double> >(std::complex<double>* to, const Packet4cd& from) {
+  EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v);
+}
+template <>
+EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double>* to, const Packet4cd& from) {
+  EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v);
+}
+
+template <>
+EIGEN_DEVICE_FUNC inline Packet4cd pgather<std::complex<double>, Packet4cd>(const std::complex<double>* from,
+                                                                            Index stride) {
+  return Packet4cd(_mm512_insertf64x4(
+      _mm512_castpd256_pd512(_mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from + 0 * stride).v),
+                                                  ploadu<Packet1cd>(from + 1 * stride).v, 1)),
+      _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from + 2 * stride).v),
+                           ploadu<Packet1cd>(from + 3 * stride).v, 1),
+      1));
+}
+
+template <>
+EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet4cd>(std::complex<double>* to, const Packet4cd& from,
+                                                                        Index stride) {
+  __m512i fromi = _mm512_castpd_si512(from.v);
+  double* tod = (double*)(void*)to;
+  _mm_storeu_pd(tod + 0 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 0)));
+  _mm_storeu_pd(tod + 2 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 1)));
+  _mm_storeu_pd(tod + 4 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 2)));
+  _mm_storeu_pd(tod + 6 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 3)));
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet4cd>(const Packet4cd& a) {
+  __m128d low = extract128<0>(a.v);
+  EIGEN_ALIGN16 double res[2];
+  _mm_store_pd(res, low);
+  return std::complex<double>(res[0], res[1]);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd preverse(const Packet4cd& a) {
+  return Packet4cd(_mm512_shuffle_f64x2(a.v, a.v, (shuffle_mask<3, 2, 1, 0>::mask)));
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<double> predux<Packet4cd>(const Packet4cd& a) {
+  return predux(padd(Packet2cd(_mm512_extractf64x4_pd(a.v, 0)), Packet2cd(_mm512_extractf64x4_pd(a.v, 1))));
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet4cd>(const Packet4cd& a) {
+  return predux_mul(pmul(Packet2cd(_mm512_extractf64x4_pd(a.v, 0)), Packet2cd(_mm512_extractf64x4_pd(a.v, 1))));
+}
+
+EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cd, Packet8d)
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd pdiv<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
+  return pdiv_complex(a, b);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd pcplxflip<Packet4cd>(const Packet4cd& x) {
+  return Packet4cd(_mm512_permute_pd(x.v, 0x55));
+}
+
+EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8cf, 4>& kernel) {
+  PacketBlock<Packet8d, 4> pb;
+
+  pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v);
+  pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v);
+  pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v);
+  pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v);
+  ptranspose(pb);
+  kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]);
+  kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]);
+  kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]);
+  kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]);
+}
+
+EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8cf, 8>& kernel) {
+  PacketBlock<Packet8d, 8> pb;
+
+  pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v);
+  pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v);
+  pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v);
+  pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v);
+  pb.packet[4] = _mm512_castps_pd(kernel.packet[4].v);
+  pb.packet[5] = _mm512_castps_pd(kernel.packet[5].v);
+  pb.packet[6] = _mm512_castps_pd(kernel.packet[6].v);
+  pb.packet[7] = _mm512_castps_pd(kernel.packet[7].v);
+  ptranspose(pb);
+  kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]);
+  kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]);
+  kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]);
+  kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]);
+  kernel.packet[4].v = _mm512_castpd_ps(pb.packet[4]);
+  kernel.packet[5].v = _mm512_castpd_ps(pb.packet[5]);
+  kernel.packet[6].v = _mm512_castpd_ps(pb.packet[6]);
+  kernel.packet[7].v = _mm512_castpd_ps(pb.packet[7]);
+}
+
+EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4cd, 4>& kernel) {
+  __m512d T0 =
+      _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<0, 1, 0, 1>::mask));  // [a0 a1 b0 b1]
+  __m512d T1 =
+      _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<2, 3, 2, 3>::mask));  // [a2 a3 b2 b3]
+  __m512d T2 =
+      _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<0, 1, 0, 1>::mask));  // [c0 c1 d0 d1]
+  __m512d T3 =
+      _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<2, 3, 2, 3>::mask));  // [c2 c3 d2 d3]
+
+  kernel.packet[3] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<1, 3, 1, 3>::mask)));  // [a3 b3 c3 d3]
+  kernel.packet[2] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<0, 2, 0, 2>::mask)));  // [a2 b2 c2 d2]
+  kernel.packet[1] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<1, 3, 1, 3>::mask)));  // [a1 b1 c1 d1]
+  kernel.packet[0] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<0, 2, 0, 2>::mask)));  // [a0 b0 c0 d0]
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd psqrt<Packet4cd>(const Packet4cd& a) {
+  return psqrt_complex<Packet4cd>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf psqrt<Packet8cf>(const Packet8cf& a) {
+  return psqrt_complex<Packet8cf>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4cd plog<Packet4cd>(const Packet4cd& a) {
+  return plog_complex<Packet4cd>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf plog<Packet8cf>(const Packet8cf& a) {
+  return plog_complex<Packet8cf>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8cf pexp<Packet8cf>(const Packet8cf& a) {
+  return pexp_complex<Packet8cf>(a);
+}
+
+}  // end namespace internal
+}  // end namespace Eigen
+
+#endif  // EIGEN_COMPLEX_AVX512_H

include/Eigen/src/Core/arch/AVX512/MathFunctions.h

@@ -0,0 +1,138 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2016 Pedro Gonnet (pedro.gonnet@gmail.com)
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_
+#define THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet16f)
+EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(Packet8d)
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pfrexp(const Packet16h& a, Packet16h& exponent) {
+  Packet16f fexponent;
+  const Packet16h out = float2half(pfrexp<Packet16f>(half2float(a), fexponent));
+  exponent = float2half(fexponent);
+  return out;
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pldexp(const Packet16h& a, const Packet16h& exponent) {
+  return float2half(pldexp<Packet16f>(half2float(a), half2float(exponent)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16bf pfrexp(const Packet16bf& a, Packet16bf& exponent) {
+  Packet16f fexponent;
+  const Packet16bf out = F32ToBf16(pfrexp<Packet16f>(Bf16ToF32(a), fexponent));
+  exponent = F32ToBf16(fexponent);
+  return out;
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16bf pldexp(const Packet16bf& a, const Packet16bf& exponent) {
+  return F32ToBf16(pldexp<Packet16f>(Bf16ToF32(a), Bf16ToF32(exponent)));
+}
+
+#if EIGEN_FAST_MATH
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16f psqrt<Packet16f>(const Packet16f& _x) {
+  return generic_sqrt_newton_step<Packet16f>::run(_x, _mm512_rsqrt14_ps(_x));
+}
+
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8d psqrt<Packet8d>(const Packet8d& _x) {
+#ifdef EIGEN_VECTORIZE_AVX512ER
+  return generic_sqrt_newton_step<Packet8d, /*Steps=*/1>::run(_x, _mm512_rsqrt28_pd(_x));
+#else
+  return generic_sqrt_newton_step<Packet8d, /*Steps=*/2>::run(_x, _mm512_rsqrt14_pd(_x));
+#endif
+}
+#else
+template <>
+EIGEN_STRONG_INLINE Packet16f psqrt<Packet16f>(const Packet16f& x) {
+  return _mm512_sqrt_ps(x);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8d psqrt<Packet8d>(const Packet8d& x) {
+  return _mm512_sqrt_pd(x);
+}
+#endif
+
+// prsqrt for float.
+#if defined(EIGEN_VECTORIZE_AVX512ER)
+template <>
+EIGEN_STRONG_INLINE Packet16f prsqrt<Packet16f>(const Packet16f& x) {
+  return _mm512_rsqrt28_ps(x);
+}
+#elif EIGEN_FAST_MATH
+
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16f prsqrt<Packet16f>(const Packet16f& _x) {
+  return generic_rsqrt_newton_step<Packet16f, /*Steps=*/1>::run(_x, _mm512_rsqrt14_ps(_x));
+}
+#endif
+
+// prsqrt for double.
+#if EIGEN_FAST_MATH
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8d prsqrt<Packet8d>(const Packet8d& _x) {
+#ifdef EIGEN_VECTORIZE_AVX512ER
+  return generic_rsqrt_newton_step<Packet8d, /*Steps=*/1>::run(_x, _mm512_rsqrt28_pd(_x));
+#else
+  return generic_rsqrt_newton_step<Packet8d, /*Steps=*/2>::run(_x, _mm512_rsqrt14_pd(_x));
+#endif
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f preciprocal<Packet16f>(const Packet16f& a) {
+#ifdef EIGEN_VECTORIZE_AVX512ER
+  return _mm512_rcp28_ps(a);
+#else
+  return generic_reciprocal_newton_step<Packet16f, /*Steps=*/1>::run(a, _mm512_rcp14_ps(a));
+#endif
+}
+#endif
+
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pcos)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pexp)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pexp2)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pexpm1)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog1p)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, plog2)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, preciprocal)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, prsqrt)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, psin)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, psqrt)
+BF16_PACKET_FUNCTION(Packet16f, Packet16bf, ptanh)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, pcos)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, pexp)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, pexp2)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, pexpm1)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, plog)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, plog1p)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, plog2)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, preciprocal)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, prsqrt)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, psin)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, psqrt)
+F16_PACKET_FUNCTION(Packet16f, Packet16h, ptanh)
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_

include/Eigen/src/Core/arch/AVX512/PacketMathFP16.h

@@ -0,0 +1,877 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+//
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_PACKET_MATH_FP16_AVX512_H
+#define EIGEN_PACKET_MATH_FP16_AVX512_H
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+typedef __m512h Packet32h;
+typedef eigen_packet_wrapper<__m256i, 1> Packet16h;
+typedef eigen_packet_wrapper<__m128i, 2> Packet8h;
+
+template <>
+struct is_arithmetic<Packet8h> {
+  enum { value = true };
+};
+
+template <>
+struct packet_traits<half> : default_packet_traits {
+  typedef Packet32h type;
+  typedef Packet16h half;
+  enum {
+    Vectorizable = 1,
+    AlignedOnScalar = 1,
+    size = 32,
+
+    HasCmp = 1,
+    HasAdd = 1,
+    HasSub = 1,
+    HasMul = 1,
+    HasDiv = 1,
+    HasNegate = 1,
+    HasAbs = 1,
+    HasAbs2 = 0,
+    HasMin = 1,
+    HasMax = 1,
+    HasConj = 1,
+    HasSetLinear = 0,
+    HasLog = 1,
+    HasLog1p = 1,
+    HasExp = 1,
+    HasExpm1 = 1,
+    HasSqrt = 1,
+    HasRsqrt = 1,
+    // These ones should be implemented in future
+    HasBessel = 0,
+    HasNdtri = 0,
+    HasSin = EIGEN_FAST_MATH,
+    HasCos = EIGEN_FAST_MATH,
+    HasTanh = EIGEN_FAST_MATH,
+    HasErf = 0,  // EIGEN_FAST_MATH,
+    HasBlend = 0
+  };
+};
+
+template <>
+struct unpacket_traits<Packet32h> {
+  typedef Eigen::half type;
+  typedef Packet16h half;
+  enum {
+    size = 32,
+    alignment = Aligned64,
+    vectorizable = true,
+    masked_load_available = false,
+    masked_store_available = false
+  };
+};
+
+template <>
+struct unpacket_traits<Packet16h> {
+  typedef Eigen::half type;
+  typedef Packet8h half;
+  enum {
+    size = 16,
+    alignment = Aligned32,
+    vectorizable = true,
+    masked_load_available = false,
+    masked_store_available = false
+  };
+};
+
+template <>
+struct unpacket_traits<Packet8h> {
+  typedef Eigen::half type;
+  typedef Packet8h half;
+  enum {
+    size = 8,
+    alignment = Aligned16,
+    vectorizable = true,
+    masked_load_available = false,
+    masked_store_available = false
+  };
+};
+
+// Memory functions
+
+// pset1
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pset1<Packet32h>(const Eigen::half& from) {
+  // half/half_raw is bit compatible
+  return _mm512_set1_ph(numext::bit_cast<_Float16>(from));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pzero(const Packet32h& /*a*/) {
+  return _mm512_setzero_ph();
+}
+
+// pset1frombits
+template <>
+EIGEN_STRONG_INLINE Packet32h pset1frombits<Packet32h>(unsigned short from) {
+  return _mm512_castsi512_ph(_mm512_set1_epi16(from));
+}
+
+// pfirst
+
+template <>
+EIGEN_STRONG_INLINE Eigen::half pfirst<Packet32h>(const Packet32h& from) {
+#ifdef EIGEN_VECTORIZE_AVX512DQ
+  return half_impl::raw_uint16_to_half(
+      static_cast<unsigned short>(_mm256_extract_epi16(_mm512_extracti32x8_epi32(_mm512_castph_si512(from), 0), 0)));
+#else
+  Eigen::half dest[32];
+  _mm512_storeu_ph(dest, from);
+  return dest[0];
+#endif
+}
+
+// pload
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pload<Packet32h>(const Eigen::half* from) {
+  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ph(from);
+}
+
+// ploadu
+
+template <>
+EIGEN_STRONG_INLINE Packet32h ploadu<Packet32h>(const Eigen::half* from) {
+  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ph(from);
+}
+
+// pstore
+
+template <>
+EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet32h& from) {
+  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ph(to, from);
+}
+
+// pstoreu
+
+template <>
+EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet32h& from) {
+  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ph(to, from);
+}
+
+// ploaddup
+template <>
+EIGEN_STRONG_INLINE Packet32h ploaddup<Packet32h>(const Eigen::half* from) {
+  __m512h a = _mm512_castph256_ph512(_mm256_loadu_ph(from));
+  return _mm512_permutexvar_ph(_mm512_set_epi16(15, 15, 14, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, 6, 6,
+                                                5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0),
+                               a);
+}
+
+// ploadquad
+template <>
+EIGEN_STRONG_INLINE Packet32h ploadquad<Packet32h>(const Eigen::half* from) {
+  __m512h a = _mm512_castph128_ph512(_mm_loadu_ph(from));
+  return _mm512_permutexvar_ph(
+      _mm512_set_epi16(7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0),
+      a);
+}
+
+// pabs
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pabs<Packet32h>(const Packet32h& a) {
+  return _mm512_abs_ph(a);
+}
+
+// psignbit
+
+template <>
+EIGEN_STRONG_INLINE Packet32h psignbit<Packet32h>(const Packet32h& a) {
+  return _mm512_castsi512_ph(_mm512_srai_epi16(_mm512_castph_si512(a), 15));
+}
+
+// pmin
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pmin<Packet32h>(const Packet32h& a, const Packet32h& b) {
+  return _mm512_min_ph(a, b);
+}
+
+// pmax
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pmax<Packet32h>(const Packet32h& a, const Packet32h& b) {
+  return _mm512_max_ph(a, b);
+}
+
+// plset
+template <>
+EIGEN_STRONG_INLINE Packet32h plset<Packet32h>(const half& a) {
+  return _mm512_add_ph(pset1<Packet32h>(a), _mm512_set_ph(31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+                                                          16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
+}
+
+// por
+
+template <>
+EIGEN_STRONG_INLINE Packet32h por(const Packet32h& a, const Packet32h& b) {
+  return _mm512_castsi512_ph(_mm512_or_si512(_mm512_castph_si512(a), _mm512_castph_si512(b)));
+}
+
+// pxor
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pxor(const Packet32h& a, const Packet32h& b) {
+  return _mm512_castsi512_ph(_mm512_xor_si512(_mm512_castph_si512(a), _mm512_castph_si512(b)));
+}
+
+// pand
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pand(const Packet32h& a, const Packet32h& b) {
+  return _mm512_castsi512_ph(_mm512_and_si512(_mm512_castph_si512(a), _mm512_castph_si512(b)));
+}
+
+// pandnot
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pandnot(const Packet32h& a, const Packet32h& b) {
+  return _mm512_castsi512_ph(_mm512_andnot_si512(_mm512_castph_si512(b), _mm512_castph_si512(a)));
+}
+
+// pselect
+
+template <>
+EIGEN_DEVICE_FUNC inline Packet32h pselect(const Packet32h& mask, const Packet32h& a, const Packet32h& b) {
+  __mmask32 mask32 = _mm512_cmp_epi16_mask(_mm512_castph_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ);
+  return _mm512_mask_blend_ph(mask32, a, b);
+}
+
+// pcmp_eq
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pcmp_eq(const Packet32h& a, const Packet32h& b) {
+  __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_EQ_OQ);
+  return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu)));
+}
+
+// pcmp_le
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pcmp_le(const Packet32h& a, const Packet32h& b) {
+  __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LE_OQ);
+  return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu)));
+}
+
+// pcmp_lt
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pcmp_lt(const Packet32h& a, const Packet32h& b) {
+  __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LT_OQ);
+  return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu)));
+}
+
+// pcmp_lt_or_nan
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pcmp_lt_or_nan(const Packet32h& a, const Packet32h& b) {
+  __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_NGE_UQ);
+  return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi16(0), mask, static_cast<short>(0xffffu)));
+}
+
+// padd
+
+template <>
+EIGEN_STRONG_INLINE Packet32h padd<Packet32h>(const Packet32h& a, const Packet32h& b) {
+  return _mm512_add_ph(a, b);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) {
+  return _mm256_castph_si256(_mm256_add_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h padd<Packet8h>(const Packet8h& a, const Packet8h& b) {
+  return _mm_castph_si128(_mm_add_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b)));
+}
+
+// psub
+
+template <>
+EIGEN_STRONG_INLINE Packet32h psub<Packet32h>(const Packet32h& a, const Packet32h& b) {
+  return _mm512_sub_ph(a, b);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h psub<Packet16h>(const Packet16h& a, const Packet16h& b) {
+  return _mm256_castph_si256(_mm256_sub_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h psub<Packet8h>(const Packet8h& a, const Packet8h& b) {
+  return _mm_castph_si128(_mm_sub_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b)));
+}
+
+// pmul
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pmul<Packet32h>(const Packet32h& a, const Packet32h& b) {
+  return _mm512_mul_ph(a, b);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) {
+  return _mm256_castph_si256(_mm256_mul_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pmul<Packet8h>(const Packet8h& a, const Packet8h& b) {
+  return _mm_castph_si128(_mm_mul_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b)));
+}
+
+// pdiv
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pdiv<Packet32h>(const Packet32h& a, const Packet32h& b) {
+  return _mm512_div_ph(a, b);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pdiv<Packet16h>(const Packet16h& a, const Packet16h& b) {
+  return _mm256_castph_si256(_mm256_div_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pdiv<Packet8h>(const Packet8h& a, const Packet8h& b) {
+  return _mm_castph_si128(_mm_div_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b)));
+}
+
+// pround
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pround<Packet32h>(const Packet32h& a) {
+  // Work-around for default std::round rounding mode.
+
+  // Mask for the sign bit
+  const Packet32h signMask = pset1frombits<Packet32h>(static_cast<numext::uint16_t>(0x8000u));
+  // The largest half-preicision float less than 0.5
+  const Packet32h prev0dot5 = pset1frombits<Packet32h>(static_cast<numext::uint16_t>(0x37FFu));
+
+  return _mm512_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO);
+}
+
+// print
+
+template <>
+EIGEN_STRONG_INLINE Packet32h print<Packet32h>(const Packet32h& a) {
+  return _mm512_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION);
+}
+
+// pceil
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pceil<Packet32h>(const Packet32h& a) {
+  return _mm512_roundscale_ph(a, _MM_FROUND_TO_POS_INF);
+}
+
+// pfloor
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pfloor<Packet32h>(const Packet32h& a) {
+  return _mm512_roundscale_ph(a, _MM_FROUND_TO_NEG_INF);
+}
+
+// ptrunc
+
+template <>
+EIGEN_STRONG_INLINE Packet32h ptrunc<Packet32h>(const Packet32h& a) {
+  return _mm512_roundscale_ph(a, _MM_FROUND_TO_ZERO);
+}
+
+// predux
+template <>
+EIGEN_STRONG_INLINE half predux<Packet32h>(const Packet32h& a) {
+  return (half)_mm512_reduce_add_ph(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE half predux<Packet16h>(const Packet16h& a) {
+  return (half)_mm256_reduce_add_ph(_mm256_castsi256_ph(a));
+}
+
+template <>
+EIGEN_STRONG_INLINE half predux<Packet8h>(const Packet8h& a) {
+  return (half)_mm_reduce_add_ph(_mm_castsi128_ph(a));
+}
+
+// predux_half_dowto4
+template <>
+EIGEN_STRONG_INLINE Packet16h predux_half_dowto4<Packet32h>(const Packet32h& a) {
+#ifdef EIGEN_VECTORIZE_AVX512DQ
+  __m256i lowHalf = _mm256_castps_si256(_mm512_extractf32x8_ps(_mm512_castph_ps(a), 0));
+  __m256i highHalf = _mm256_castps_si256(_mm512_extractf32x8_ps(_mm512_castph_ps(a), 1));
+
+  return Packet16h(padd<Packet16h>(lowHalf, highHalf));
+#else
+  Eigen::half data[32];
+  _mm512_storeu_ph(data, a);
+
+  __m256i lowHalf = _mm256_castph_si256(_mm256_loadu_ph(data));
+  __m256i highHalf = _mm256_castph_si256(_mm256_loadu_ph(data + 16));
+
+  return Packet16h(padd<Packet16h>(lowHalf, highHalf));
+#endif
+}
+
+// predux_max
+
+// predux_min
+
+// predux_mul
+
+#ifdef EIGEN_VECTORIZE_FMA
+
+// pmadd
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pmadd(const Packet32h& a, const Packet32h& b, const Packet32h& c) {
+  return _mm512_fmadd_ph(a, b, c);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pmadd(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
+  return _mm256_castph_si256(_mm256_fmadd_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b), _mm256_castsi256_ph(c)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pmadd(const Packet8h& a, const Packet8h& b, const Packet8h& c) {
+  return _mm_castph_si128(_mm_fmadd_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b), _mm_castsi128_ph(c)));
+}
+
+// pmsub
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pmsub(const Packet32h& a, const Packet32h& b, const Packet32h& c) {
+  return _mm512_fmsub_ph(a, b, c);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pmsub(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
+  return _mm256_castph_si256(_mm256_fmsub_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b), _mm256_castsi256_ph(c)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pmsub(const Packet8h& a, const Packet8h& b, const Packet8h& c) {
+  return _mm_castph_si128(_mm_fmsub_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b), _mm_castsi128_ph(c)));
+}
+
+// pnmadd
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pnmadd(const Packet32h& a, const Packet32h& b, const Packet32h& c) {
+  return _mm512_fnmadd_ph(a, b, c);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pnmadd(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
+  return _mm256_castph_si256(_mm256_fnmadd_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b), _mm256_castsi256_ph(c)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pnmadd(const Packet8h& a, const Packet8h& b, const Packet8h& c) {
+  return _mm_castph_si128(_mm_fnmadd_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b), _mm_castsi128_ph(c)));
+}
+
+// pnmsub
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pnmsub(const Packet32h& a, const Packet32h& b, const Packet32h& c) {
+  return _mm512_fnmsub_ph(a, b, c);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pnmsub(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
+  return _mm256_castph_si256(_mm256_fnmsub_ph(_mm256_castsi256_ph(a), _mm256_castsi256_ph(b), _mm256_castsi256_ph(c)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pnmsub(const Packet8h& a, const Packet8h& b, const Packet8h& c) {
+  return _mm_castph_si128(_mm_fnmsub_ph(_mm_castsi128_ph(a), _mm_castsi128_ph(b), _mm_castsi128_ph(c)));
+}
+
+#endif
+
+// pnegate
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pnegate<Packet32h>(const Packet32h& a) {
+  return psub(pzero(a), a);
+}
+
+// pconj
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pconj<Packet32h>(const Packet32h& a) {
+  return a;
+}
+
+// psqrt
+
+template <>
+EIGEN_STRONG_INLINE Packet32h psqrt<Packet32h>(const Packet32h& a) {
+  return _mm512_sqrt_ph(a);
+}
+
+// prsqrt
+
+template <>
+EIGEN_STRONG_INLINE Packet32h prsqrt<Packet32h>(const Packet32h& a) {
+  return _mm512_rsqrt_ph(a);
+}
+
+// preciprocal
+
+template <>
+EIGEN_STRONG_INLINE Packet32h preciprocal<Packet32h>(const Packet32h& a) {
+  return _mm512_rcp_ph(a);
+}
+
+// ptranspose
+
+EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet32h, 32>& a) {
+  __m512i t[32];
+
+  EIGEN_UNROLL_LOOP
+  for (int i = 0; i < 16; i++) {
+    t[2 * i] = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1]));
+    t[2 * i + 1] =
+        _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1]));
+  }
+
+  __m512i p[32];
+
+  EIGEN_UNROLL_LOOP
+  for (int i = 0; i < 8; i++) {
+    p[4 * i] = _mm512_unpacklo_epi32(t[4 * i], t[4 * i + 2]);
+    p[4 * i + 1] = _mm512_unpackhi_epi32(t[4 * i], t[4 * i + 2]);
+    p[4 * i + 2] = _mm512_unpacklo_epi32(t[4 * i + 1], t[4 * i + 3]);
+    p[4 * i + 3] = _mm512_unpackhi_epi32(t[4 * i + 1], t[4 * i + 3]);
+  }
+
+  __m512i q[32];
+
+  EIGEN_UNROLL_LOOP
+  for (int i = 0; i < 4; i++) {
+    q[8 * i] = _mm512_unpacklo_epi64(p[8 * i], p[8 * i + 4]);
+    q[8 * i + 1] = _mm512_unpackhi_epi64(p[8 * i], p[8 * i + 4]);
+    q[8 * i + 2] = _mm512_unpacklo_epi64(p[8 * i + 1], p[8 * i + 5]);
+    q[8 * i + 3] = _mm512_unpackhi_epi64(p[8 * i + 1], p[8 * i + 5]);
+    q[8 * i + 4] = _mm512_unpacklo_epi64(p[8 * i + 2], p[8 * i + 6]);
+    q[8 * i + 5] = _mm512_unpackhi_epi64(p[8 * i + 2], p[8 * i + 6]);
+    q[8 * i + 6] = _mm512_unpacklo_epi64(p[8 * i + 3], p[8 * i + 7]);
+    q[8 * i + 7] = _mm512_unpackhi_epi64(p[8 * i + 3], p[8 * i + 7]);
+  }
+
+  __m512i f[32];
+
+#define PACKET32H_TRANSPOSE_HELPER(X, Y)                                                            \
+  do {                                                                                              \
+    f[Y * 8] = _mm512_inserti32x4(f[Y * 8], _mm512_extracti32x4_epi32(q[X * 8], Y), X);             \
+    f[Y * 8 + 1] = _mm512_inserti32x4(f[Y * 8 + 1], _mm512_extracti32x4_epi32(q[X * 8 + 1], Y), X); \
+    f[Y * 8 + 2] = _mm512_inserti32x4(f[Y * 8 + 2], _mm512_extracti32x4_epi32(q[X * 8 + 2], Y), X); \
+    f[Y * 8 + 3] = _mm512_inserti32x4(f[Y * 8 + 3], _mm512_extracti32x4_epi32(q[X * 8 + 3], Y), X); \
+    f[Y * 8 + 4] = _mm512_inserti32x4(f[Y * 8 + 4], _mm512_extracti32x4_epi32(q[X * 8 + 4], Y), X); \
+    f[Y * 8 + 5] = _mm512_inserti32x4(f[Y * 8 + 5], _mm512_extracti32x4_epi32(q[X * 8 + 5], Y), X); \
+    f[Y * 8 + 6] = _mm512_inserti32x4(f[Y * 8 + 6], _mm512_extracti32x4_epi32(q[X * 8 + 6], Y), X); \
+    f[Y * 8 + 7] = _mm512_inserti32x4(f[Y * 8 + 7], _mm512_extracti32x4_epi32(q[X * 8 + 7], Y), X); \
+  } while (false);
+
+  PACKET32H_TRANSPOSE_HELPER(0, 0);
+  PACKET32H_TRANSPOSE_HELPER(1, 1);
+  PACKET32H_TRANSPOSE_HELPER(2, 2);
+  PACKET32H_TRANSPOSE_HELPER(3, 3);
+
+  PACKET32H_TRANSPOSE_HELPER(1, 0);
+  PACKET32H_TRANSPOSE_HELPER(2, 0);
+  PACKET32H_TRANSPOSE_HELPER(3, 0);
+  PACKET32H_TRANSPOSE_HELPER(2, 1);
+  PACKET32H_TRANSPOSE_HELPER(3, 1);
+  PACKET32H_TRANSPOSE_HELPER(3, 2);
+
+  PACKET32H_TRANSPOSE_HELPER(0, 1);
+  PACKET32H_TRANSPOSE_HELPER(0, 2);
+  PACKET32H_TRANSPOSE_HELPER(0, 3);
+  PACKET32H_TRANSPOSE_HELPER(1, 2);
+  PACKET32H_TRANSPOSE_HELPER(1, 3);
+  PACKET32H_TRANSPOSE_HELPER(2, 3);
+
+#undef PACKET32H_TRANSPOSE_HELPER
+
+  EIGEN_UNROLL_LOOP
+  for (int i = 0; i < 32; i++) {
+    a.packet[i] = _mm512_castsi512_ph(f[i]);
+  }
+}
+
+EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet32h, 4>& a) {
+  __m512i p0, p1, p2, p3, t0, t1, t2, t3, a0, a1, a2, a3;
+  t0 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1]));
+  t1 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1]));
+  t2 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3]));
+  t3 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3]));
+
+  p0 = _mm512_unpacklo_epi32(t0, t2);
+  p1 = _mm512_unpackhi_epi32(t0, t2);
+  p2 = _mm512_unpacklo_epi32(t1, t3);
+  p3 = _mm512_unpackhi_epi32(t1, t3);
+
+  a0 = p0;
+  a1 = p1;
+  a2 = p2;
+  a3 = p3;
+
+  a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p1, 0), 1);
+  a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p0, 1), 0);
+
+  a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p2, 0), 2);
+  a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p0, 2), 0);
+
+  a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p3, 0), 3);
+  a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p0, 3), 0);
+
+  a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p2, 1), 2);
+  a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p1, 2), 1);
+
+  a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p3, 2), 3);
+  a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p2, 3), 2);
+
+  a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p3, 1), 3);
+  a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p1, 3), 1);
+
+  a.packet[0] = _mm512_castsi512_ph(a0);
+  a.packet[1] = _mm512_castsi512_ph(a1);
+  a.packet[2] = _mm512_castsi512_ph(a2);
+  a.packet[3] = _mm512_castsi512_ph(a3);
+}
+
+// preverse
+
+template <>
+EIGEN_STRONG_INLINE Packet32h preverse(const Packet32h& a) {
+  return _mm512_permutexvar_ph(_mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+                                                20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),
+                               a);
+}
+
+// pscatter
+
+template <>
+EIGEN_STRONG_INLINE void pscatter<half, Packet32h>(half* to, const Packet32h& from, Index stride) {
+  EIGEN_ALIGN64 half aux[32];
+  pstore(aux, from);
+
+  EIGEN_UNROLL_LOOP
+  for (int i = 0; i < 32; i++) {
+    to[stride * i] = aux[i];
+  }
+}
+
+// pgather
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pgather<Eigen::half, Packet32h>(const Eigen::half* from, Index stride) {
+  return _mm512_castsi512_ph(_mm512_set_epi16(
+      from[31 * stride].x, from[30 * stride].x, from[29 * stride].x, from[28 * stride].x, from[27 * stride].x,
+      from[26 * stride].x, from[25 * stride].x, from[24 * stride].x, from[23 * stride].x, from[22 * stride].x,
+      from[21 * stride].x, from[20 * stride].x, from[19 * stride].x, from[18 * stride].x, from[17 * stride].x,
+      from[16 * stride].x, from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x,
+      from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x, from[7 * stride].x,
+      from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, from[3 * stride].x, from[2 * stride].x,
+      from[1 * stride].x, from[0 * stride].x));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pcos<Packet16h>(const Packet16h&);
+template <>
+EIGEN_STRONG_INLINE Packet16h psin<Packet16h>(const Packet16h&);
+template <>
+EIGEN_STRONG_INLINE Packet16h plog<Packet16h>(const Packet16h&);
+template <>
+EIGEN_STRONG_INLINE Packet16h plog2<Packet16h>(const Packet16h&);
+template <>
+EIGEN_STRONG_INLINE Packet16h plog1p<Packet16h>(const Packet16h&);
+template <>
+EIGEN_STRONG_INLINE Packet16h pexp<Packet16h>(const Packet16h&);
+template <>
+EIGEN_STRONG_INLINE Packet16h pexpm1<Packet16h>(const Packet16h&);
+template <>
+EIGEN_STRONG_INLINE Packet16h ptanh<Packet16h>(const Packet16h&);
+template <>
+EIGEN_STRONG_INLINE Packet16h pfrexp<Packet16h>(const Packet16h&, Packet16h&);
+template <>
+EIGEN_STRONG_INLINE Packet16h pldexp<Packet16h>(const Packet16h&, const Packet16h&);
+
+EIGEN_STRONG_INLINE Packet32h combine2Packet16h(const Packet16h& a, const Packet16h& b) {
+  __m512d result = _mm512_undefined_pd();
+  result = _mm512_insertf64x4(result, _mm256_castsi256_pd(a), 0);
+  result = _mm512_insertf64x4(result, _mm256_castsi256_pd(b), 1);
+  return _mm512_castpd_ph(result);
+}
+
+EIGEN_STRONG_INLINE void extract2Packet16h(const Packet32h& x, Packet16h& a, Packet16h& b) {
+  a = _mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castph_pd(x), 0));
+  b = _mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castph_pd(x), 1));
+}
+
+// psin
+template <>
+EIGEN_STRONG_INLINE Packet32h psin<Packet32h>(const Packet32h& a) {
+  Packet16h low;
+  Packet16h high;
+  extract2Packet16h(a, low, high);
+
+  Packet16h lowOut = psin(low);
+  Packet16h highOut = psin(high);
+
+  return combine2Packet16h(lowOut, highOut);
+}
+
+// pcos
+template <>
+EIGEN_STRONG_INLINE Packet32h pcos<Packet32h>(const Packet32h& a) {
+  Packet16h low;
+  Packet16h high;
+  extract2Packet16h(a, low, high);
+
+  Packet16h lowOut = pcos(low);
+  Packet16h highOut = pcos(high);
+
+  return combine2Packet16h(lowOut, highOut);
+}
+
+// plog
+template <>
+EIGEN_STRONG_INLINE Packet32h plog<Packet32h>(const Packet32h& a) {
+  Packet16h low;
+  Packet16h high;
+  extract2Packet16h(a, low, high);
+
+  Packet16h lowOut = plog(low);
+  Packet16h highOut = plog(high);
+
+  return combine2Packet16h(lowOut, highOut);
+}
+
+// plog2
+template <>
+EIGEN_STRONG_INLINE Packet32h plog2<Packet32h>(const Packet32h& a) {
+  Packet16h low;
+  Packet16h high;
+  extract2Packet16h(a, low, high);
+
+  Packet16h lowOut = plog2(low);
+  Packet16h highOut = plog2(high);
+
+  return combine2Packet16h(lowOut, highOut);
+}
+
+// plog1p
+template <>
+EIGEN_STRONG_INLINE Packet32h plog1p<Packet32h>(const Packet32h& a) {
+  Packet16h low;
+  Packet16h high;
+  extract2Packet16h(a, low, high);
+
+  Packet16h lowOut = plog1p(low);
+  Packet16h highOut = plog1p(high);
+
+  return combine2Packet16h(lowOut, highOut);
+}
+
+// pexp
+template <>
+EIGEN_STRONG_INLINE Packet32h pexp<Packet32h>(const Packet32h& a) {
+  Packet16h low;
+  Packet16h high;
+  extract2Packet16h(a, low, high);
+
+  Packet16h lowOut = pexp(low);
+  Packet16h highOut = pexp(high);
+
+  return combine2Packet16h(lowOut, highOut);
+}
+
+// pexpm1
+template <>
+EIGEN_STRONG_INLINE Packet32h pexpm1<Packet32h>(const Packet32h& a) {
+  Packet16h low;
+  Packet16h high;
+  extract2Packet16h(a, low, high);
+
+  Packet16h lowOut = pexpm1(low);
+  Packet16h highOut = pexpm1(high);
+
+  return combine2Packet16h(lowOut, highOut);
+}
+
+// ptanh
+template <>
+EIGEN_STRONG_INLINE Packet32h ptanh<Packet32h>(const Packet32h& a) {
+  Packet16h low;
+  Packet16h high;
+  extract2Packet16h(a, low, high);
+
+  Packet16h lowOut = ptanh(low);
+  Packet16h highOut = ptanh(high);
+
+  return combine2Packet16h(lowOut, highOut);
+}
+
+// pfrexp
+template <>
+EIGEN_STRONG_INLINE Packet32h pfrexp<Packet32h>(const Packet32h& a, Packet32h& exponent) {
+  Packet16h low;
+  Packet16h high;
+  extract2Packet16h(a, low, high);
+
+  Packet16h exp1 = _mm256_undefined_si256();
+  Packet16h exp2 = _mm256_undefined_si256();
+
+  Packet16h lowOut = pfrexp(low, exp1);
+  Packet16h highOut = pfrexp(high, exp2);
+
+  exponent = combine2Packet16h(exp1, exp2);
+
+  return combine2Packet16h(lowOut, highOut);
+}
+
+// pldexp
+template <>
+EIGEN_STRONG_INLINE Packet32h pldexp<Packet32h>(const Packet32h& a, const Packet32h& exponent) {
+  Packet16h low;
+  Packet16h high;
+  extract2Packet16h(a, low, high);
+
+  Packet16h exp1;
+  Packet16h exp2;
+  extract2Packet16h(exponent, exp1, exp2);
+
+  Packet16h lowOut = pldexp(low, exp1);
+  Packet16h highOut = pldexp(high, exp2);
+
+  return combine2Packet16h(lowOut, highOut);
+}
+
+}  // end namespace internal
+}  // end namespace Eigen
+
+#endif  // EIGEN_PACKET_MATH_FP16_AVX512_H

include/Eigen/src/Core/arch/AVX512/TypeCasting.h

@@ -0,0 +1,336 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2019 Rasmus Munk Larsen <rmlarsen@google.com>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_TYPE_CASTING_AVX512_H
+#define EIGEN_TYPE_CASTING_AVX512_H
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <>
+struct type_casting_traits<float, bool> : vectorized_type_casting_traits<float, bool> {};
+template <>
+struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {};
+
+template <>
+struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {};
+template <>
+struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {};
+
+template <>
+struct type_casting_traits<float, double> : vectorized_type_casting_traits<float, double> {};
+template <>
+struct type_casting_traits<double, float> : vectorized_type_casting_traits<double, float> {};
+
+template <>
+struct type_casting_traits<double, int> : vectorized_type_casting_traits<double, int> {};
+template <>
+struct type_casting_traits<int, double> : vectorized_type_casting_traits<int, double> {};
+
+template <>
+struct type_casting_traits<double, int64_t> : vectorized_type_casting_traits<double, int64_t> {};
+template <>
+struct type_casting_traits<int64_t, double> : vectorized_type_casting_traits<int64_t, double> {};
+
+template <>
+struct type_casting_traits<half, float> : vectorized_type_casting_traits<half, float> {};
+template <>
+struct type_casting_traits<float, half> : vectorized_type_casting_traits<float, half> {};
+
+template <>
+struct type_casting_traits<bfloat16, float> : vectorized_type_casting_traits<bfloat16, float> {};
+template <>
+struct type_casting_traits<float, bfloat16> : vectorized_type_casting_traits<float, bfloat16> {};
+
+template <>
+EIGEN_STRONG_INLINE Packet16b pcast<Packet16f, Packet16b>(const Packet16f& a) {
+  __mmask16 mask = _mm512_cmpneq_ps_mask(a, pzero(a));
+  return _mm512_maskz_cvtepi32_epi8(mask, _mm512_set1_epi32(1));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f pcast<Packet16b, Packet16f>(const Packet16b& a) {
+  return _mm512_cvtepi32_ps(_mm512_and_si512(_mm512_cvtepi8_epi32(a), _mm512_set1_epi32(1)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16i pcast<Packet16f, Packet16i>(const Packet16f& a) {
+  return _mm512_cvttps_epi32(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8d pcast<Packet16f, Packet8d>(const Packet16f& a) {
+  return _mm512_cvtps_pd(_mm512_castps512_ps256(a));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8d pcast<Packet8f, Packet8d>(const Packet8f& a) {
+  return _mm512_cvtps_pd(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8l pcast<Packet8d, Packet8l>(const Packet8d& a) {
+#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVX512VL)
+  return _mm512_cvttpd_epi64(a);
+#else
+  constexpr int kTotalBits = sizeof(double) * CHAR_BIT, kMantissaBits = std::numeric_limits<double>::digits - 1,
+                kExponentBits = kTotalBits - kMantissaBits - 1, kBias = (1 << (kExponentBits - 1)) - 1;
+
+  const __m512i cst_one = _mm512_set1_epi64(1);
+  const __m512i cst_total_bits = _mm512_set1_epi64(kTotalBits);
+  const __m512i cst_bias = _mm512_set1_epi64(kBias);
+
+  __m512i a_bits = _mm512_castpd_si512(a);
+  // shift left by 1 to clear the sign bit, and shift right by kMantissaBits + 1 to recover biased exponent
+  __m512i biased_e = _mm512_srli_epi64(_mm512_slli_epi64(a_bits, 1), kMantissaBits + 1);
+  __m512i e = _mm512_sub_epi64(biased_e, cst_bias);
+
+  // shift to the left by kExponentBits + 1 to clear the sign and exponent bits
+  __m512i shifted_mantissa = _mm512_slli_epi64(a_bits, kExponentBits + 1);
+  // shift to the right by kTotalBits - e to convert the significand to an integer
+  __m512i result_significand = _mm512_srlv_epi64(shifted_mantissa, _mm512_sub_epi64(cst_total_bits, e));
+
+  // add the implied bit
+  __m512i result_exponent = _mm512_sllv_epi64(cst_one, e);
+  // e <= 0 is interpreted as a large positive shift (2's complement), which also conveniently results in zero
+  __m512i result = _mm512_add_epi64(result_significand, result_exponent);
+  // handle negative arguments
+  __mmask8 sign_mask = _mm512_cmplt_epi64_mask(a_bits, _mm512_setzero_si512());
+  result = _mm512_mask_sub_epi64(result, sign_mask, _mm512_setzero_si512(), result);
+  return result;
+#endif
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f pcast<Packet16i, Packet16f>(const Packet16i& a) {
+  return _mm512_cvtepi32_ps(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8d pcast<Packet16i, Packet8d>(const Packet16i& a) {
+  return _mm512_cvtepi32_pd(_mm512_castsi512_si256(a));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8d pcast<Packet8i, Packet8d>(const Packet8i& a) {
+  return _mm512_cvtepi32_pd(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8d pcast<Packet8l, Packet8d>(const Packet8l& a) {
+#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVX512VL)
+  return _mm512_cvtepi64_pd(a);
+#else
+  EIGEN_ALIGN64 int64_t aux[8];
+  pstore(aux, a);
+  return _mm512_set_pd(static_cast<double>(aux[7]), static_cast<double>(aux[6]), static_cast<double>(aux[5]),
+                       static_cast<double>(aux[4]), static_cast<double>(aux[3]), static_cast<double>(aux[2]),
+                       static_cast<double>(aux[1]), static_cast<double>(aux[0]));
+#endif
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f pcast<Packet8d, Packet16f>(const Packet8d& a, const Packet8d& b) {
+  return cat256(_mm512_cvtpd_ps(a), _mm512_cvtpd_ps(b));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16i pcast<Packet8d, Packet16i>(const Packet8d& a, const Packet8d& b) {
+  return cat256i(_mm512_cvttpd_epi32(a), _mm512_cvttpd_epi32(b));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8i pcast<Packet8d, Packet8i>(const Packet8d& a) {
+  return _mm512_cvtpd_epi32(a);
+}
+template <>
+EIGEN_STRONG_INLINE Packet8f pcast<Packet8d, Packet8f>(const Packet8d& a) {
+  return _mm512_cvtpd_ps(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16i preinterpret<Packet16i, Packet16f>(const Packet16f& a) {
+  return _mm512_castps_si512(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet16i>(const Packet16i& a) {
+  return _mm512_castsi512_ps(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet16f>(const Packet16f& a) {
+  return _mm512_castps_pd(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet8l>(const Packet8l& a) {
+  return _mm512_castsi512_pd(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8l preinterpret<Packet8l, Packet8d>(const Packet8d& a) {
+  return _mm512_castpd_si512(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet8d>(const Packet8d& a) {
+  return _mm512_castpd_ps(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8f preinterpret<Packet8f, Packet16f>(const Packet16f& a) {
+  return _mm512_castps512_ps256(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet16f>(const Packet16f& a) {
+  return _mm512_castps512_ps128(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4d preinterpret<Packet4d, Packet8d>(const Packet8d& a) {
+  return _mm512_castpd512_pd256(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet8d>(const Packet8d& a) {
+  return _mm512_castpd512_pd128(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet8f>(const Packet8f& a) {
+  return _mm512_castps256_ps512(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet4f>(const Packet4f& a) {
+  return _mm512_castps128_ps512(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet4d>(const Packet4d& a) {
+  return _mm512_castpd256_pd512(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet2d>(const Packet2d& a) {
+  return _mm512_castpd128_pd512(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8i preinterpret<Packet8i, Packet16i>(const Packet16i& a) {
+  return _mm512_castsi512_si256(a);
+}
+template <>
+EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet16i>(const Packet16i& a) {
+  return _mm512_castsi512_si128(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h preinterpret<Packet8h, Packet16h>(const Packet16h& a) {
+  return _mm256_castsi256_si128(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8bf preinterpret<Packet8bf, Packet16bf>(const Packet16bf& a) {
+  return _mm256_castsi256_si128(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f pcast<Packet16h, Packet16f>(const Packet16h& a) {
+  return half2float(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pcast<Packet16f, Packet16h>(const Packet16f& a) {
+  return float2half(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f pcast<Packet16bf, Packet16f>(const Packet16bf& a) {
+  return Bf16ToF32(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16bf pcast<Packet16f, Packet16bf>(const Packet16f& a) {
+  return F32ToBf16(a);
+}
+
+#ifdef EIGEN_VECTORIZE_AVX512FP16
+
+template <>
+EIGEN_STRONG_INLINE Packet16h preinterpret<Packet16h, Packet32h>(const Packet32h& a) {
+  return _mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castph_pd(a), 0));
+}
+template <>
+EIGEN_STRONG_INLINE Packet8h preinterpret<Packet8h, Packet32h>(const Packet32h& a) {
+  return _mm256_castsi256_si128(preinterpret<Packet16h>(a));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16f pcast<Packet32h, Packet16f>(const Packet32h& a) {
+  // Discard second-half of input.
+  Packet16h low = _mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castph_pd(a), 0));
+  return _mm512_cvtxph_ps(_mm256_castsi256_ph(low));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet32h pcast<Packet16f, Packet32h>(const Packet16f& a, const Packet16f& b) {
+  __m512d result = _mm512_undefined_pd();
+  result = _mm512_insertf64x4(
+      result, _mm256_castsi256_pd(_mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)), 0);
+  result = _mm512_insertf64x4(
+      result, _mm256_castsi256_pd(_mm512_cvtps_ph(b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)), 1);
+  return _mm512_castpd_ph(result);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8f pcast<Packet16h, Packet8f>(const Packet16h& a) {
+  // Discard second-half of input.
+  Packet8h low = _mm_castps_si128(_mm256_extractf32x4_ps(_mm256_castsi256_ps(a), 0));
+  return _mm256_cvtxph_ps(_mm_castsi128_ph(low));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet16h pcast<Packet8f, Packet16h>(const Packet8f& a, const Packet8f& b) {
+  __m256d result = _mm256_undefined_pd();
+  result = _mm256_insertf64x2(result,
+                              _mm_castsi128_pd(_mm256_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)), 0);
+  result = _mm256_insertf64x2(result,
+                              _mm_castsi128_pd(_mm256_cvtps_ph(b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)), 1);
+  return _mm256_castpd_si256(result);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4f pcast<Packet8h, Packet4f>(const Packet8h& a) {
+  Packet8f full = _mm256_cvtxph_ps(_mm_castsi128_ph(a));
+  // Discard second-half of input.
+  return _mm256_extractf32x4_ps(full, 0);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8h pcast<Packet4f, Packet8h>(const Packet4f& a, const Packet4f& b) {
+  __m256 result = _mm256_undefined_ps();
+  result = _mm256_insertf128_ps(result, a, 0);
+  result = _mm256_insertf128_ps(result, b, 1);
+  return _mm256_cvtps_ph(result, _MM_FROUND_TO_NEAREST_INT);
+}
+
+#endif
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_TYPE_CASTING_AVX512_H

include/Eigen/src/Core/arch/AltiVec/Complex.h

@@ -0,0 +1,644 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2010 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2010-2016 Konstantinos Margaritis <markos@freevec.org>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_COMPLEX32_ALTIVEC_H
+#define EIGEN_COMPLEX32_ALTIVEC_H
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+static Packet4ui p4ui_CONJ_XOR =
+    vec_mergeh((Packet4ui)p4i_ZERO, (Packet4ui)p4f_MZERO);  //{ 0x00000000, 0x80000000, 0x00000000, 0x80000000 };
+#ifdef EIGEN_VECTORIZE_VSX
+#if defined(_BIG_ENDIAN)
+static Packet2ul p2ul_CONJ_XOR1 =
+    (Packet2ul)vec_sld((Packet4ui)p2d_MZERO, (Packet4ui)p2l_ZERO, 8);  //{ 0x8000000000000000, 0x0000000000000000 };
+static Packet2ul p2ul_CONJ_XOR2 =
+    (Packet2ul)vec_sld((Packet4ui)p2l_ZERO, (Packet4ui)p2d_MZERO, 8);  //{ 0x8000000000000000, 0x0000000000000000 };
+#else
+static Packet2ul p2ul_CONJ_XOR1 =
+    (Packet2ul)vec_sld((Packet4ui)p2l_ZERO, (Packet4ui)p2d_MZERO, 8);  //{ 0x8000000000000000, 0x0000000000000000 };
+static Packet2ul p2ul_CONJ_XOR2 =
+    (Packet2ul)vec_sld((Packet4ui)p2d_MZERO, (Packet4ui)p2l_ZERO, 8);  //{ 0x8000000000000000, 0x0000000000000000 };
+#endif
+#endif
+
+//---------- float ----------
+struct Packet2cf {
+  EIGEN_STRONG_INLINE explicit Packet2cf() {}
+  EIGEN_STRONG_INLINE explicit Packet2cf(const Packet4f& a) : v(a) {}
+
+  EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) {
+    Packet4f v1, v2;
+
+    // Permute and multiply the real parts of a and b
+    v1 = vec_perm(a.v, a.v, p16uc_PSET32_WODD);
+    // Get the imaginary parts of a
+    v2 = vec_perm(a.v, a.v, p16uc_PSET32_WEVEN);
+    // multiply a_re * b
+    v1 = vec_madd(v1, b.v, p4f_ZERO);
+    // multiply a_im * b and get the conjugate result
+    v2 = vec_madd(v2, b.v, p4f_ZERO);
+    v2 = reinterpret_cast<Packet4f>(pxor(v2, reinterpret_cast<Packet4f>(p4ui_CONJ_XOR)));
+    // permute back to a proper order
+    v2 = vec_perm(v2, v2, p16uc_COMPLEX32_REV);
+
+    return Packet2cf(padd<Packet4f>(v1, v2));
+  }
+
+  EIGEN_STRONG_INLINE Packet2cf& operator*=(const Packet2cf& b) {
+    v = pmul(Packet2cf(*this), b).v;
+    return *this;
+  }
+  EIGEN_STRONG_INLINE Packet2cf operator*(const Packet2cf& b) const { return Packet2cf(*this) *= b; }
+
+  EIGEN_STRONG_INLINE Packet2cf& operator+=(const Packet2cf& b) {
+    v = padd(v, b.v);
+    return *this;
+  }
+  EIGEN_STRONG_INLINE Packet2cf operator+(const Packet2cf& b) const { return Packet2cf(*this) += b; }
+  EIGEN_STRONG_INLINE Packet2cf& operator-=(const Packet2cf& b) {
+    v = psub(v, b.v);
+    return *this;
+  }
+  EIGEN_STRONG_INLINE Packet2cf operator-(const Packet2cf& b) const { return Packet2cf(*this) -= b; }
+  EIGEN_STRONG_INLINE Packet2cf operator-(void) const { return Packet2cf(-v); }
+
+  Packet4f v;
+};
+
+template <>
+struct packet_traits<std::complex<float> > : default_packet_traits {
+  typedef Packet2cf type;
+  typedef Packet2cf half;
+  typedef Packet4f as_real;
+  enum {
+    Vectorizable = 1,
+    AlignedOnScalar = 1,
+    size = 2,
+
+    HasAdd = 1,
+    HasSub = 1,
+    HasMul = 1,
+    HasDiv = 1,
+    HasNegate = 1,
+    HasAbs = 0,
+    HasAbs2 = 0,
+    HasMin = 0,
+    HasMax = 0,
+    HasSqrt = 1,
+    HasLog = 1,
+    HasExp = 1,
+#ifdef EIGEN_VECTORIZE_VSX
+    HasBlend = 1,
+#endif
+    HasSetLinear = 0
+  };
+};
+
+template <>
+struct unpacket_traits<Packet2cf> {
+  typedef std::complex<float> type;
+  enum {
+    size = 2,
+    alignment = Aligned16,
+    vectorizable = true,
+    masked_load_available = false,
+    masked_store_available = false
+  };
+  typedef Packet2cf half;
+  typedef Packet4f as_real;
+};
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<float>& from) {
+  Packet2cf res;
+#ifdef EIGEN_VECTORIZE_VSX
+  // Load a single std::complex<float> from memory and duplicate
+  //
+  // Using pload would read past the end of the reference in this case
+  // Using vec_xl_len + vec_splat, generates poor assembly
+  __asm__("lxvdsx %x0,%y1" : "=wa"(res.v) : "Z"(from));
+#else
+  if ((std::ptrdiff_t(&from) % 16) == 0)
+    res.v = pload<Packet4f>((const float*)&from);
+  else
+    res.v = ploadu<Packet4f>((const float*)&from);
+  res.v = vec_perm(res.v, res.v, p16uc_PSET64_HI);
+#endif
+  return res;
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf pload<Packet2cf>(const std::complex<float>* from) {
+  return Packet2cf(pload<Packet4f>((const float*)from));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) {
+  return Packet2cf(ploadu<Packet4f>((const float*)from));
+}
+template <>
+EIGEN_ALWAYS_INLINE Packet2cf pload_partial<Packet2cf>(const std::complex<float>* from, const Index n,
+                                                       const Index offset) {
+  return Packet2cf(pload_partial<Packet4f>((const float*)from, n * 2, offset * 2));
+}
+template <>
+EIGEN_ALWAYS_INLINE Packet2cf ploadu_partial<Packet2cf>(const std::complex<float>* from, const Index n,
+                                                        const Index offset) {
+  return Packet2cf(ploadu_partial<Packet4f>((const float*)from, n * 2, offset * 2));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cf ploaddup<Packet2cf>(const std::complex<float>* from) {
+  return pset1<Packet2cf>(*from);
+}
+
+template <>
+EIGEN_STRONG_INLINE void pstore<std::complex<float> >(std::complex<float>* to, const Packet2cf& from) {
+  pstore((float*)to, from.v);
+}
+template <>
+EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet2cf& from) {
+  pstoreu((float*)to, from.v);
+}
+template <>
+EIGEN_ALWAYS_INLINE void pstore_partial<std::complex<float> >(std::complex<float>* to, const Packet2cf& from,
+                                                              const Index n, const Index offset) {
+  pstore_partial((float*)to, from.v, n * 2, offset * 2);
+}
+template <>
+EIGEN_ALWAYS_INLINE void pstoreu_partial<std::complex<float> >(std::complex<float>* to, const Packet2cf& from,
+                                                               const Index n, const Index offset) {
+  pstoreu_partial((float*)to, from.v, n * 2, offset * 2);
+}
+
+EIGEN_STRONG_INLINE Packet2cf pload2(const std::complex<float>& from0, const std::complex<float>& from1) {
+  Packet4f res0, res1;
+#ifdef EIGEN_VECTORIZE_VSX
+  // Load two std::complex<float> from memory and combine
+  __asm__("lxsdx %x0,%y1" : "=wa"(res0) : "Z"(from0));
+  __asm__("lxsdx %x0,%y1" : "=wa"(res1) : "Z"(from1));
+#ifdef _BIG_ENDIAN
+  __asm__("xxpermdi %x0, %x1, %x2, 0" : "=wa"(res0) : "wa"(res0), "wa"(res1));
+#else
+  __asm__("xxpermdi %x0, %x2, %x1, 0" : "=wa"(res0) : "wa"(res0), "wa"(res1));
+#endif
+#else
+  *reinterpret_cast<std::complex<float>*>(&res0) = from0;
+  *reinterpret_cast<std::complex<float>*>(&res1) = from1;
+  res0 = vec_perm(res0, res1, p16uc_TRANSPOSE64_HI);
+#endif
+  return Packet2cf(res0);
+}
+
+template <>
+EIGEN_ALWAYS_INLINE Packet2cf pload_ignore<Packet2cf>(const std::complex<float>* from) {
+  Packet2cf res;
+  res.v = pload_ignore<Packet4f>(reinterpret_cast<const float*>(from));
+  return res;
+}
+
+template <typename Scalar, typename Packet>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pgather_complex_size2(const Scalar* from, Index stride,
+                                                                   const Index n = 2) {
+  eigen_internal_assert(n <= unpacket_traits<Packet>::size && "number of elements will gather past end of packet");
+  EIGEN_ALIGN16 Scalar af[2];
+  for (Index i = 0; i < n; i++) {
+    af[i] = from[i * stride];
+  }
+  return pload_ignore<Packet>(af);
+}
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2cf pgather<std::complex<float>, Packet2cf>(const std::complex<float>* from,
+                                                                                        Index stride) {
+  return pgather_complex_size2<std::complex<float>, Packet2cf>(from, stride);
+}
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2cf
+pgather_partial<std::complex<float>, Packet2cf>(const std::complex<float>* from, Index stride, const Index n) {
+  return pgather_complex_size2<std::complex<float>, Packet2cf>(from, stride, n);
+}
+template <typename Scalar, typename Packet>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_complex_size2(Scalar* to, const Packet& from, Index stride,
+                                                                  const Index n = 2) {
+  eigen_internal_assert(n <= unpacket_traits<Packet>::size && "number of elements will scatter past end of packet");
+  EIGEN_ALIGN16 Scalar af[2];
+  pstore<Scalar>((Scalar*)af, from);
+  for (Index i = 0; i < n; i++) {
+    to[i * stride] = af[i];
+  }
+}
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<std::complex<float>, Packet2cf>(std::complex<float>* to,
+                                                                                    const Packet2cf& from,
+                                                                                    Index stride) {
+  pscatter_complex_size2<std::complex<float>, Packet2cf>(to, from, stride);
+}
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial<std::complex<float>, Packet2cf>(std::complex<float>* to,
+                                                                                            const Packet2cf& from,
+                                                                                            Index stride,
+                                                                                            const Index n) {
+  pscatter_complex_size2<std::complex<float>, Packet2cf>(to, from, stride, n);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf padd<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
+  return Packet2cf(a.v + b.v);
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cf psub<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
+  return Packet2cf(a.v - b.v);
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) {
+  return Packet2cf(pnegate(a.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) {
+  return Packet2cf(pxor<Packet4f>(a.v, reinterpret_cast<Packet4f>(p4ui_CONJ_XOR)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf pand<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
+  return Packet2cf(pand<Packet4f>(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cf por<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
+  return Packet2cf(por<Packet4f>(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cf pxor<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
+  return Packet2cf(pxor<Packet4f>(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
+  return Packet2cf(pandnot<Packet4f>(a.v, b.v));
+}
+
+template <>
+EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float>* addr) {
+  EIGEN_PPC_PREFETCH(addr);
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2cf>(const Packet2cf& a) {
+  EIGEN_ALIGN16 std::complex<float> res[2];
+  pstore((float*)&res, a.v);
+
+  return res[0];
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) {
+  Packet4f rev_a;
+  rev_a = vec_sld(a.v, a.v, 8);
+  return Packet2cf(rev_a);
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<float> predux<Packet2cf>(const Packet2cf& a) {
+  Packet4f b;
+  b = vec_sld(a.v, a.v, 8);
+  b = padd<Packet4f>(a.v, b);
+  return pfirst<Packet2cf>(Packet2cf(b));
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet2cf>(const Packet2cf& a) {
+  Packet4f b;
+  Packet2cf prod;
+  b = vec_sld(a.v, a.v, 8);
+  prod = pmul<Packet2cf>(a, Packet2cf(b));
+
+  return pfirst<Packet2cf>(prod);
+}
+
+EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf, Packet4f)
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
+  return pdiv_complex(a, b);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf pcplxflip<Packet2cf>(const Packet2cf& x) {
+  return Packet2cf(vec_perm(x.v, x.v, p16uc_COMPLEX32_REV));
+}
+
+EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2cf, 2>& kernel) {
+#ifdef EIGEN_VECTORIZE_VSX
+  Packet4f tmp = reinterpret_cast<Packet4f>(
+      vec_mergeh(reinterpret_cast<Packet2d>(kernel.packet[0].v), reinterpret_cast<Packet2d>(kernel.packet[1].v)));
+  kernel.packet[1].v = reinterpret_cast<Packet4f>(
+      vec_mergel(reinterpret_cast<Packet2d>(kernel.packet[0].v), reinterpret_cast<Packet2d>(kernel.packet[1].v)));
+#else
+  Packet4f tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI);
+  kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO);
+#endif
+  kernel.packet[0].v = tmp;
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packet2cf& b) {
+  Packet4f eq = reinterpret_cast<Packet4f>(vec_cmpeq(a.v, b.v));
+  return Packet2cf(vec_and(eq, vec_perm(eq, eq, p16uc_COMPLEX32_REV)));
+}
+
+#ifdef EIGEN_VECTORIZE_VSX
+template <>
+EIGEN_STRONG_INLINE Packet2cf pblend(const Selector<2>& ifPacket, const Packet2cf& thenPacket,
+                                     const Packet2cf& elsePacket) {
+  Packet2cf result;
+  result.v = reinterpret_cast<Packet4f>(
+      pblend<Packet2d>(ifPacket, reinterpret_cast<Packet2d>(thenPacket.v), reinterpret_cast<Packet2d>(elsePacket.v)));
+  return result;
+}
+#endif
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf psqrt<Packet2cf>(const Packet2cf& a) {
+  return psqrt_complex<Packet2cf>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf plog<Packet2cf>(const Packet2cf& a) {
+  return plog_complex<Packet2cf>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet2cf pexp<Packet2cf>(const Packet2cf& a) {
+  return pexp_complex<Packet2cf>(a);
+}
+
+//---------- double ----------
+#ifdef EIGEN_VECTORIZE_VSX
+struct Packet1cd {
+  EIGEN_STRONG_INLINE Packet1cd() {}
+  EIGEN_STRONG_INLINE explicit Packet1cd(const Packet2d& a) : v(a) {}
+
+  EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) {
+    Packet2d a_re, a_im, v1, v2;
+
+    // Permute and multiply the real parts of a and b
+    a_re = vec_perm(a.v, a.v, p16uc_PSET64_HI);
+    // Get the imaginary parts of a
+    a_im = vec_perm(a.v, a.v, p16uc_PSET64_LO);
+    // multiply a_re * b
+    v1 = vec_madd(a_re, b.v, p2d_ZERO);
+    // multiply a_im * b and get the conjugate result
+    v2 = vec_madd(a_im, b.v, p2d_ZERO);
+    v2 = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(v2), reinterpret_cast<Packet4ui>(v2), 8));
+    v2 = pxor(v2, reinterpret_cast<Packet2d>(p2ul_CONJ_XOR1));
+
+    return Packet1cd(padd<Packet2d>(v1, v2));
+  }
+
+  EIGEN_STRONG_INLINE Packet1cd& operator*=(const Packet1cd& b) {
+    v = pmul(Packet1cd(*this), b).v;
+    return *this;
+  }
+  EIGEN_STRONG_INLINE Packet1cd operator*(const Packet1cd& b) const { return Packet1cd(*this) *= b; }
+
+  EIGEN_STRONG_INLINE Packet1cd& operator+=(const Packet1cd& b) {
+    v = padd(v, b.v);
+    return *this;
+  }
+  EIGEN_STRONG_INLINE Packet1cd operator+(const Packet1cd& b) const { return Packet1cd(*this) += b; }
+  EIGEN_STRONG_INLINE Packet1cd& operator-=(const Packet1cd& b) {
+    v = psub(v, b.v);
+    return *this;
+  }
+  EIGEN_STRONG_INLINE Packet1cd operator-(const Packet1cd& b) const { return Packet1cd(*this) -= b; }
+  EIGEN_STRONG_INLINE Packet1cd operator-(void) const { return Packet1cd(-v); }
+
+  Packet2d v;
+};
+
+template <>
+struct packet_traits<std::complex<double> > : default_packet_traits {
+  typedef Packet1cd type;
+  typedef Packet1cd half;
+  typedef Packet2d as_real;
+  enum {
+    Vectorizable = 1,
+    AlignedOnScalar = 0,
+    size = 1,
+
+    HasAdd = 1,
+    HasSub = 1,
+    HasMul = 1,
+    HasDiv = 1,
+    HasNegate = 1,
+    HasAbs = 0,
+    HasAbs2 = 0,
+    HasMin = 0,
+    HasMax = 0,
+    HasSqrt = 1,
+    HasLog = 1,
+    HasSetLinear = 0
+  };
+};
+
+template <>
+struct unpacket_traits<Packet1cd> {
+  typedef std::complex<double> type;
+  enum {
+    size = 1,
+    alignment = Aligned16,
+    vectorizable = true,
+    masked_load_available = false,
+    masked_store_available = false
+  };
+  typedef Packet1cd half;
+  typedef Packet2d as_real;
+};
+
+template <>
+EIGEN_STRONG_INLINE Packet1cd pload<Packet1cd>(const std::complex<double>* from) {
+  return Packet1cd(pload<Packet2d>((const double*)from));
+}
+template <>
+EIGEN_STRONG_INLINE Packet1cd ploadu<Packet1cd>(const std::complex<double>* from) {
+  return Packet1cd(ploadu<Packet2d>((const double*)from));
+}
+template <>
+EIGEN_ALWAYS_INLINE Packet1cd pload_partial<Packet1cd>(const std::complex<double>* from, const Index n,
+                                                       const Index offset) {
+  return Packet1cd(pload_partial<Packet2d>((const double*)from, n * 2, offset * 2));
+}
+template <>
+EIGEN_ALWAYS_INLINE Packet1cd ploadu_partial<Packet1cd>(const std::complex<double>* from, const Index n,
+                                                        const Index offset) {
+  return Packet1cd(ploadu_partial<Packet2d>((const double*)from, n * 2, offset * 2));
+}
+template <>
+EIGEN_STRONG_INLINE void pstore<std::complex<double> >(std::complex<double>* to, const Packet1cd& from) {
+  pstore((double*)to, from.v);
+}
+template <>
+EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double>* to, const Packet1cd& from) {
+  pstoreu((double*)to, from.v);
+}
+template <>
+EIGEN_ALWAYS_INLINE void pstore_partial<std::complex<double> >(std::complex<double>* to, const Packet1cd& from,
+                                                               const Index n, const Index offset) {
+  pstore_partial((double*)to, from.v, n * 2, offset * 2);
+}
+template <>
+EIGEN_ALWAYS_INLINE void pstoreu_partial<std::complex<double> >(std::complex<double>* to, const Packet1cd& from,
+                                                                const Index n, const Index offset) {
+  pstoreu_partial((double*)to, from.v, n * 2, offset * 2);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet1cd
+pset1<Packet1cd>(const std::complex<double>& from) { /* here we really have to use unaligned loads :( */
+  return ploadu<Packet1cd>(&from);
+}
+
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet1cd
+pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from, Index) {
+  return pload<Packet1cd>(from);
+}
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet1cd
+pgather_partial<std::complex<double>, Packet1cd>(const std::complex<double>* from, Index, const Index) {
+  return pload<Packet1cd>(from);
+}
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<std::complex<double>, Packet1cd>(std::complex<double>* to,
+                                                                                     const Packet1cd& from, Index) {
+  pstore<std::complex<double> >(to, from);
+}
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial<std::complex<double>, Packet1cd>(std::complex<double>* to,
+                                                                                             const Packet1cd& from,
+                                                                                             Index, const Index) {
+  pstore<std::complex<double> >(to, from);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet1cd padd<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
+  return Packet1cd(a.v + b.v);
+}
+template <>
+EIGEN_STRONG_INLINE Packet1cd psub<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
+  return Packet1cd(a.v - b.v);
+}
+template <>
+EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) {
+  return Packet1cd(pnegate(Packet2d(a.v)));
+}
+template <>
+EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) {
+  return Packet1cd(pxor(a.v, reinterpret_cast<Packet2d>(p2ul_CONJ_XOR2)));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet1cd pand<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
+  return Packet1cd(pand(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet1cd por<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
+  return Packet1cd(por(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet1cd pxor<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
+  return Packet1cd(pxor(a.v, b.v));
+}
+template <>
+EIGEN_STRONG_INLINE Packet1cd pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
+  return Packet1cd(pandnot(a.v, b.v));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet1cd ploaddup<Packet1cd>(const std::complex<double>* from) {
+  return pset1<Packet1cd>(*from);
+}
+
+template <>
+EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double>* addr) {
+  EIGEN_PPC_PREFETCH(addr);
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet1cd>(const Packet1cd& a) {
+  EIGEN_ALIGN16 std::complex<double> res[1];
+  pstore<std::complex<double> >(res, a);
+
+  return res[0];
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) {
+  return a;
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<double> predux<Packet1cd>(const Packet1cd& a) {
+  return pfirst(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet1cd>(const Packet1cd& a) {
+  return pfirst(a);
+}
+
+EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd, Packet2d)
+
+template <>
+EIGEN_STRONG_INLINE Packet1cd pdiv<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
+  return pdiv_complex(a, b);
+}
+
+EIGEN_STRONG_INLINE Packet1cd pcplxflip /*<Packet1cd>*/ (const Packet1cd& x) {
+  return Packet1cd(preverse(Packet2d(x.v)));
+}
+
+EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet1cd, 2>& kernel) {
+  Packet2d tmp = vec_mergeh(kernel.packet[0].v, kernel.packet[1].v);
+  kernel.packet[1].v = vec_mergel(kernel.packet[0].v, kernel.packet[1].v);
+  kernel.packet[0].v = tmp;
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet1cd pcmp_eq(const Packet1cd& a, const Packet1cd& b) {
+  // Compare real and imaginary parts of a and b to get the mask vector:
+  // [re(a)==re(b), im(a)==im(b)]
+  Packet2d eq = reinterpret_cast<Packet2d>(vec_cmpeq(a.v, b.v));
+  // Swap real/imag elements in the mask in to get:
+  // [im(a)==im(b), re(a)==re(b)]
+  Packet2d eq_swapped =
+      reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(eq), reinterpret_cast<Packet4ui>(eq), 8));
+  // Return re(a)==re(b) & im(a)==im(b) by computing bitwise AND of eq and eq_swapped
+  return Packet1cd(vec_and(eq, eq_swapped));
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet1cd psqrt<Packet1cd>(const Packet1cd& a) {
+  return psqrt_complex<Packet1cd>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet1cd plog<Packet1cd>(const Packet1cd& a) {
+  return plog_complex<Packet1cd>(a);
+}
+
+#endif  // __VSX__
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_COMPLEX32_ALTIVEC_H

include/Eigen/src/Core/arch/AltiVec/MathFunctions.h

@@ -0,0 +1,81 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2007 Julien Pommier
+// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2016 Konstantinos Margaritis <markos@freevec.org>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_MATH_FUNCTIONS_ALTIVEC_H
+#define EIGEN_MATH_FUNCTIONS_ALTIVEC_H
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet4f)
+#ifdef EIGEN_VECTORIZE_VSX
+EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(Packet2d)
+#endif
+
+#ifdef EIGEN_VECTORIZE_VSX
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f psqrt<Packet4f>(const Packet4f& x) {
+  return vec_sqrt(x);
+}
+
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d psqrt<Packet2d>(const Packet2d& x) {
+  return vec_sqrt(x);
+}
+
+#if !EIGEN_COMP_CLANG
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f prsqrt<Packet4f>(const Packet4f& x) {
+  return pset1<Packet4f>(1.0f) / psqrt<Packet4f>(x);
+  //  vec_rsqrt returns different results from the generic version
+  //  return  vec_rsqrt(x);
+}
+
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d prsqrt<Packet2d>(const Packet2d& x) {
+  return pset1<Packet2d>(1.0) / psqrt<Packet2d>(x);
+  //  vec_rsqrt returns different results from the generic version
+  //  return  vec_rsqrt(x);
+}
+
+#endif
+
+template <>
+EIGEN_STRONG_INLINE Packet8bf psqrt<Packet8bf>(const Packet8bf& a) {
+  BF16_TO_F32_UNARY_OP_WRAPPER(psqrt<Packet4f>, a);
+}
+
+#if !EIGEN_COMP_CLANG
+template <>
+EIGEN_STRONG_INLINE Packet8bf prsqrt<Packet8bf>(const Packet8bf& a) {
+  BF16_TO_F32_UNARY_OP_WRAPPER(prsqrt<Packet4f>, a);
+}
+#endif
+#else
+template <>
+EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f psqrt<Packet4f>(const Packet4f& x) {
+  Packet4f a;
+  for (Index i = 0; i < packet_traits<float>::size; i++) {
+    a[i] = numext::sqrt(x[i]);
+  }
+  return a;
+}
+#endif
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_MATH_FUNCTIONS_ALTIVEC_H

include/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h

@@ -0,0 +1,205 @@
+// #define EIGEN_POWER_USE_PREFETCH  // Use prefetching in gemm routines
+#ifdef EIGEN_POWER_USE_PREFETCH
+#define EIGEN_POWER_PREFETCH(p) prefetch(p)
+#else
+#define EIGEN_POWER_PREFETCH(p)
+#endif
+
+#if defined(_ARCH_PWR9) || defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
+#define USE_PARTIAL_PACKETS
+#endif
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+
+template <typename Scalar, typename Packet, typename DataMapper, const Index accRows, const Index accCols>
+EIGEN_ALWAYS_INLINE void gemm_extra_row(const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base,
+                                        Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows,
+                                        Index remaining_rows, const Packet& pAlpha, const Packet& pMask);
+
+template <typename Scalar, typename Packet, typename DataMapper, const Index accCols>
+EIGEN_ALWAYS_INLINE void gemm_extra_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index depth,
+                                         Index strideA, Index offsetA, Index strideB, Index offsetB, Index col,
+                                         Index rows, Index cols, Index remaining_rows, const Packet& pAlpha,
+                                         const Packet& pMask);
+
+template <typename Packet>
+EIGEN_ALWAYS_INLINE Packet bmask(const Index remaining_rows);
+
+template <typename Scalar, typename Packet, typename Packetc, typename DataMapper, const Index accRows,
+          const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
+EIGEN_ALWAYS_INLINE void gemm_complex_extra_row(const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base,
+                                                Index depth, Index strideA, Index offsetA, Index strideB, Index row,
+                                                Index rows, Index remaining_rows, const Packet& pAlphaReal,
+                                                const Packet& pAlphaImag, const Packet& pMask);
+
+template <typename Scalar, typename Packet, typename Packetc, typename DataMapper, const Index accCols,
+          bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
+EIGEN_ALWAYS_INLINE void gemm_complex_extra_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB,
+                                                 Index depth, Index strideA, Index offsetA, Index strideB,
+                                                 Index offsetB, Index col, Index rows, Index cols, Index remaining_rows,
+                                                 const Packet& pAlphaReal, const Packet& pAlphaImag,
+                                                 const Packet& pMask);
+
+template <typename DataMapper>
+EIGEN_ALWAYS_INLINE void convertArrayBF16toF32(float* result, Index cols, Index rows, const DataMapper& src);
+
+template <const Index size, bool non_unit_stride, Index delta>
+EIGEN_ALWAYS_INLINE void storeBF16fromResult(bfloat16* dst, Packet8bf data, Index resInc, Index extra = 0);
+
+template <bool non_unit_stride = false>
+EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32(float* result, Index cols, Index rows, bfloat16* src,
+                                                      Index resInc = 1);
+
+template <bool rhsExtraCols, bool lhsExtraRows>
+EIGEN_ALWAYS_INLINE void storeResults(Packet4f (&acc)[4], Index rows, const Packet4f pAlpha, float* result,
+                                      Index extra_cols, Index extra_rows);
+
+template <Index num_acc, bool extraRows, Index size = 4>
+EIGEN_ALWAYS_INLINE void outputVecColResults(Packet4f (&acc)[num_acc][size], float* result, Packet4f pAlpha,
+                                             Index extra_rows);
+
+template <Index num_acc, Index size = 4>
+EIGEN_ALWAYS_INLINE void outputVecResults(Packet4f (&acc)[num_acc][size], float* result, Packet4f pAlpha);
+
+template <typename RhsMapper, bool linear>
+EIGEN_ALWAYS_INLINE Packet8bf loadColData(RhsMapper& rhs, Index j);
+
+template <typename Packet>
+EIGEN_ALWAYS_INLINE Packet ploadLhs(const __UNPACK_TYPE__(Packet) * lhs);
+
+template <typename DataMapper, typename Packet, const Index accCols, int StorageOrder, bool Complex, int N,
+          bool full = true>
+EIGEN_ALWAYS_INLINE void bload(PacketBlock<Packet, N*(Complex ? 2 : 1)>& acc, const DataMapper& res, Index row,
+                               Index col);
+
+template <typename DataMapper, typename Packet, int N>
+EIGEN_ALWAYS_INLINE void bstore(PacketBlock<Packet, N>& acc, const DataMapper& res, Index row);
+
+#ifdef USE_PARTIAL_PACKETS
+template <typename DataMapper, typename Packet, const Index accCols, bool Complex, Index N, bool full = true>
+EIGEN_ALWAYS_INLINE void bload_partial(PacketBlock<Packet, N*(Complex ? 2 : 1)>& acc, const DataMapper& res, Index row,
+                                       Index elements);
+
+template <typename DataMapper, typename Packet, Index N>
+EIGEN_ALWAYS_INLINE void bstore_partial(PacketBlock<Packet, N>& acc, const DataMapper& res, Index row, Index elements);
+#endif
+
+template <typename Packet, int N>
+EIGEN_ALWAYS_INLINE void bscale(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ, const Packet& pAlpha);
+
+template <typename Packet, int N, bool mask>
+EIGEN_ALWAYS_INLINE void bscale(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ, const Packet& pAlpha,
+                                const Packet& pMask);
+
+template <typename Packet, int N, bool mask>
+EIGEN_ALWAYS_INLINE void bscalec(PacketBlock<Packet, N>& aReal, PacketBlock<Packet, N>& aImag, const Packet& bReal,
+                                 const Packet& bImag, PacketBlock<Packet, N>& cReal, PacketBlock<Packet, N>& cImag,
+                                 const Packet& pMask);
+
+template <typename Packet, typename Packetc, int N, bool full>
+EIGEN_ALWAYS_INLINE void bcouple(PacketBlock<Packet, N>& taccReal, PacketBlock<Packet, N>& taccImag,
+                                 PacketBlock<Packetc, N * 2>& tRes, PacketBlock<Packetc, N>& acc1,
+                                 PacketBlock<Packetc, N>& acc2);
+
+#define MICRO_NORMAL(iter) (accCols == accCols2) || (unroll_factor != (iter + 1))
+
+#define MICRO_UNROLL_ITER1(func, N)          \
+  switch (remaining_rows) {                  \
+    default:                                 \
+      func(N, 0) break;                      \
+    case 1:                                  \
+      func(N, 1) break;                      \
+    case 2:                                  \
+      if (sizeof(Scalar) == sizeof(float)) { \
+        func(N, 2)                           \
+      }                                      \
+      break;                                 \
+    case 3:                                  \
+      if (sizeof(Scalar) == sizeof(float)) { \
+        func(N, 3)                           \
+      }                                      \
+      break;                                 \
+  }
+
+#ifdef USE_PARTIAL_PACKETS
+#define MICRO_UNROLL_ITER(func, N) \
+  if (remaining_rows) {            \
+    func(N, true);                 \
+  } else {                         \
+    func(N, false);                \
+  }
+
+#define MICRO_NORMAL_PARTIAL(iter) full || (unroll_factor != (iter + 1))
+#else
+#define MICRO_UNROLL_ITER(func, N) MICRO_UNROLL_ITER1(func, N)
+#endif
+
+#define MICRO_COMPLEX_UNROLL_ITER(func, N) MICRO_UNROLL_ITER1(func, N)
+
+#define MICRO_NORMAL_COLS(iter, a, b) ((MICRO_NORMAL(iter)) ? a : b)
+
+#define MICRO_LOAD1(lhs_ptr, iter)                               \
+  if (unroll_factor > iter) {                                    \
+    lhsV##iter = ploadLhs<Packet>(lhs_ptr##iter);                \
+    lhs_ptr##iter += MICRO_NORMAL_COLS(iter, accCols, accCols2); \
+  } else {                                                       \
+    EIGEN_UNUSED_VARIABLE(lhsV##iter);                           \
+  }
+
+#define MICRO_LOAD_ONE(iter) MICRO_LOAD1(lhs_ptr, iter)
+
+#define MICRO_COMPLEX_LOAD_ONE(iter)                                                                       \
+  if (!LhsIsReal && (unroll_factor > iter)) {                                                              \
+    lhsVi##iter = ploadLhs<Packet>(lhs_ptr_real##iter + MICRO_NORMAL_COLS(iter, imag_delta, imag_delta2)); \
+  } else {                                                                                                 \
+    EIGEN_UNUSED_VARIABLE(lhsVi##iter);                                                                    \
+  }                                                                                                        \
+  MICRO_LOAD1(lhs_ptr_real, iter)
+
+#define MICRO_SRC_PTR1(lhs_ptr, advRows, iter)                                  \
+  if (unroll_factor > iter) {                                                   \
+    lhs_ptr##iter = lhs_base + (row + (iter * accCols)) * strideA * advRows -   \
+                    MICRO_NORMAL_COLS(iter, 0, (accCols - accCols2) * offsetA); \
+  } else {                                                                      \
+    EIGEN_UNUSED_VARIABLE(lhs_ptr##iter);                                       \
+  }
+
+#define MICRO_SRC_PTR_ONE(iter) MICRO_SRC_PTR1(lhs_ptr, 1, iter)
+
+#define MICRO_COMPLEX_SRC_PTR_ONE(iter) MICRO_SRC_PTR1(lhs_ptr_real, advanceRows, iter)
+
+#define MICRO_PREFETCH1(lhs_ptr, iter)   \
+  if (unroll_factor > iter) {            \
+    EIGEN_POWER_PREFETCH(lhs_ptr##iter); \
+  }
+
+#define MICRO_PREFETCH_ONE(iter) MICRO_PREFETCH1(lhs_ptr, iter)
+
+#define MICRO_COMPLEX_PREFETCH_ONE(iter) MICRO_PREFETCH1(lhs_ptr_real, iter)
+
+#ifdef USE_PARTIAL_PACKETS
+#define MICRO_UPDATE_MASK
+#else
+#define MICRO_UPDATE_MASK EIGEN_UNUSED_VARIABLE(pMask);
+#endif
+
+#define MICRO_UPDATE                \
+  if (accCols == accCols2) {        \
+    MICRO_UPDATE_MASK               \
+    EIGEN_UNUSED_VARIABLE(offsetA); \
+    row += unroll_factor * accCols; \
+  }
+
+#define MICRO_COMPLEX_UPDATE                \
+  MICRO_UPDATE                              \
+  if (LhsIsReal || (accCols == accCols2)) { \
+    EIGEN_UNUSED_VARIABLE(imag_delta2);     \
+  }
+
+}  // end namespace internal
+}  // end namespace Eigen

include/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h

@@ -0,0 +1,901 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2020 Everton Constantino (everton.constantino@ibm.com)
+// Copyright (C) 2021 Chip Kerchner (chip.kerchner@ibm.com)
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
+#define EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
+
+// If using dynamic dispatch, set the CPU target.
+#if defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
+#pragma GCC push_options
+#pragma GCC target("cpu=power10,htm")
+#endif
+
+#ifdef __has_builtin
+#if !__has_builtin(__builtin_vsx_assemble_pair)
+#define __builtin_vsx_assemble_pair __builtin_mma_assemble_pair
+#endif
+#if !__has_builtin(__builtin_vsx_disassemble_pair)
+#define __builtin_vsx_disassemble_pair __builtin_mma_disassemble_pair
+#endif
+#endif
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+#include "MatrixProductMMAbfloat16.h"
+
+namespace Eigen {
+
+namespace internal {
+
+#define accColsC (accCols / 2)
+
+EIGEN_ALWAYS_INLINE void bsetzeroMMA(__vector_quad* acc) { __builtin_mma_xxsetaccz(acc); }
+
+template <typename DataMapper, typename Packet, bool full>
+EIGEN_ALWAYS_INLINE void storeAccumulator(Index i, const DataMapper& data, const Packet& alpha, const Index elements,
+                                          __vector_quad* acc) {
+  PacketBlock<Packet, 4> result;
+  __builtin_mma_disassemble_acc(&result.packet, acc);
+
+  PacketBlock<Packet, 4> tRes;
+  if (full) {
+    EIGEN_UNUSED_VARIABLE(elements);
+    bload<DataMapper, Packet, 0, ColMajor, false, 4>(tRes, data, i, 0);
+    bscale<Packet, 4>(tRes, result, alpha);
+    bstore<DataMapper, Packet, 4>(tRes, data, i);
+  } else {
+    bload_partial<DataMapper, Packet, 0, false, 4>(tRes, data, i, elements);
+    bscale<Packet, 4>(tRes, result, alpha);
+    bstore_partial<DataMapper, Packet, 4>(tRes, data, i, elements);
+  }
+}
+
+template <typename DataMapper, typename Packet, typename Packetc, const Index accCols, const Index accCols2>
+EIGEN_ALWAYS_INLINE void storeComplexAccumulator(Index i, const DataMapper& data, const Packet& alphaReal,
+                                                 const Packet& alphaImag, const Packet& pMask, __vector_quad* accReal,
+                                                 __vector_quad* accImag) {
+  constexpr bool full = (accCols2 > accColsC);
+  PacketBlock<Packet, 4> resultReal, resultImag;
+  __builtin_mma_disassemble_acc(&resultReal.packet, accReal);
+  __builtin_mma_disassemble_acc(&resultImag.packet, accImag);
+
+  PacketBlock<Packetc, 8> tRes;
+  bload<DataMapper, Packetc, accColsC, ColMajor, true, 4, full>(tRes, data, i, 0);
+
+  PacketBlock<Packet, 4> taccReal, taccImag;
+  bscalec<Packet, 4, (accCols != accCols2)>(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag, pMask);
+
+  PacketBlock<Packetc, 4> acc1, acc2;
+  bcouple<Packet, Packetc, 4, full>(taccReal, taccImag, tRes, acc1, acc2);
+
+  bstore<DataMapper, Packetc, 4>(acc1, data, i);
+  if (full) {
+    bstore<DataMapper, Packetc, 4>(acc2, data, i + accColsC);
+  }
+}
+
+// Defaults to float32, since Eigen still supports C++03 we can't use default template arguments
+template <typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
+EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const RhsPacket& a, const LhsPacket& b) {
+  if (NegativeAccumulate) {
+    __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
+  } else {
+    __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
+  }
+}
+
+template <typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
+EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const __vector_pair& a, const Packet2d& b) {
+  if (NegativeAccumulate) {
+    __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b);
+  } else {
+    __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b);
+  }
+}
+
+template <typename Packet, typename RhsPacket, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
+EIGEN_ALWAYS_INLINE void pgercMMA(__vector_quad* accReal, __vector_quad* accImag, const Packet& lhsV, Packet& lhsVi,
+                                  const RhsPacket& rhsV, RhsPacket& rhsVi) {
+  pgerMMA<Packet, RhsPacket, false>(accReal, rhsV, lhsV);
+  if (LhsIsReal) {
+    pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi, lhsV);
+    EIGEN_UNUSED_VARIABLE(lhsVi);
+  } else {
+    if (!RhsIsReal) {
+      pgerMMA<Packet, RhsPacket, ConjugateLhs == ConjugateRhs>(accReal, rhsVi, lhsVi);
+      pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi, lhsV);
+    } else {
+      EIGEN_UNUSED_VARIABLE(rhsVi);
+    }
+    pgerMMA<Packet, RhsPacket, ConjugateLhs>(accImag, rhsV, lhsVi);
+  }
+}
+
+// This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled.
+template <typename Packet>
+EIGEN_ALWAYS_INLINE Packet ploadRhs(const __UNPACK_TYPE__(Packet) * rhs) {
+  return ploadu<Packet>(rhs);
+}
+
+template <typename Scalar, typename Packet>
+EIGEN_ALWAYS_INLINE void ploadRhsMMA(const Scalar* rhs, Packet& rhsV) {
+  rhsV = ploadRhs<Packet>(rhs);
+}
+
+template <>
+EIGEN_ALWAYS_INLINE void ploadRhsMMA(const double* rhs, __vector_pair& rhsV) {
+#if EIGEN_COMP_LLVM
+  __builtin_vsx_assemble_pair(
+      &rhsV, reinterpret_cast<__vector unsigned char>(ploadRhs<Packet2d>(rhs + (sizeof(Packet2d) / sizeof(double)))),
+      reinterpret_cast<__vector unsigned char>(ploadRhs<Packet2d>(rhs)));
+#else
+  rhsV = *reinterpret_cast<__vector_pair*>(const_cast<double*>(rhs));
+#endif
+}
+
+EIGEN_ALWAYS_INLINE void ploadLhsMMA(const double* lhs, __vector_pair& lhsV) { ploadRhsMMA(lhs, lhsV); }
+
+#define GEMM_MULTIPLE_COLS
+
+// Disable in GCC until unnecessary register moves are fixed
+// #if (EIGEN_COMP_LLVM || (__GNUC__ >= 11))
+#if EIGEN_COMP_LLVM
+#define VECTOR_PAIR_LOADS_LHS
+#endif
+
+// PEEL_MMA loop factor.
+#ifdef GEMM_MULTIPLE_COLS
+#define PEEL_MMA 8
+#else
+// Register spillage with GCC12+
+#if EIGEN_COMP_LLVM || (__GNUC__ < 12) || defined(VECTOR_PAIR_LOADS_LHS)
+#define PEEL_MMA 7
+#else
+#define PEEL_MMA 6
+#endif
+#endif
+
+#define MICRO_MMA_UNROLL(func) func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)
+
+#define MICRO_MMA_WORK(func, type, peel)                                                                        \
+  if (accItr == 1) {                                                                                            \
+    func(0, type, peel, 0, 0) func(1, type, peel, 1, 0) func(2, type, peel, 2, 0) func(3, type, peel, 3, 0)     \
+        func(4, type, peel, 4, 0) func(5, type, peel, 5, 0) func(6, type, peel, 6, 0) func(7, type, peel, 7, 0) \
+  } else if (accItr == 2) {                                                                                     \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 1, 0) func(3, type, peel, 1, 1)     \
+        func(4, type, peel, 2, 0) func(5, type, peel, 2, 1) func(6, type, peel, 3, 0) func(7, type, peel, 3, 1) \
+  } else {                                                                                                      \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 0, 2) func(3, type, peel, 0, 3)     \
+        func(4, type, peel, 1, 0) func(5, type, peel, 1, 1) func(6, type, peel, 1, 2) func(7, type, peel, 1, 3) \
+  }
+
+#define MICRO_MMA_WORK_ONE(iter, type, peel, left, right)                        \
+  if (unroll_factor > left) {                                                    \
+    pgerMMA<Packet, type, false>(&accZero##iter, rhsV##right[peel], lhsV##left); \
+  }
+
+#ifdef VECTOR_PAIR_LOADS_LHS
+#define MICRO_MMA_WORK_TWO(iter, type, peel, left, right)                                          \
+  if (unroll_factor > left) {                                                                      \
+    pgerMMA<Packet, type, false>(&accZero##iter, rhsV##right[peel], lhsV2##left.packet[peel & 1]); \
+  }
+
+#define MICRO_MMA_LOAD1_TWO(lhs_ptr, left)                                                        \
+  if (unroll_factor > left) {                                                                     \
+    if (MICRO_NORMAL(left)) {                                                                     \
+      ploadLhsMMA(reinterpret_cast<const double*>(lhs_ptr##left), plhsV##left);                   \
+      __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&lhsV2##left.packet), &plhsV##left); \
+      lhs_ptr##left += accCols * 2;                                                               \
+    } else {                                                                                      \
+      lhsV2##left.packet[0] = ploadLhs<Packet>(lhs_ptr##left);                                    \
+      lhsV2##left.packet[1] = ploadLhs<Packet>(lhs_ptr##left + accCols2);                         \
+      lhs_ptr##left += accCols2 * 2;                                                              \
+      EIGEN_UNUSED_VARIABLE(plhsV##left);                                                         \
+    }                                                                                             \
+  } else {                                                                                        \
+    EIGEN_UNUSED_VARIABLE(lhsV2##left);                                                           \
+    EIGEN_UNUSED_VARIABLE(plhsV##left);                                                           \
+  }
+
+#define MICRO_MMA_LOAD_TWO(left) MICRO_MMA_LOAD1_TWO(lhs_ptr, left)
+#endif
+
+#define MICRO_MMA_UNROLL_ITER(func, val)                       \
+  func(val, 0) if (accItr > 1) {                               \
+    func(val, 1) if (accItr > 2) { func(val, 2) func(val, 3) } \
+  }
+
+#define MICRO_MMA_LOAD_ONE_RHS1(peel, right) ploadRhsMMA(rhs_ptr##right + (accRows * peel), rhsV##right[peel]);
+
+#define MICRO_MMA_LOAD_ONE_RHS(peel) MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_ONE_RHS1, peel)
+
+#define MICRO_MMA_TYPE_PEEL(funcw, funcl, type, peel)              \
+  if (PEEL_MMA > peel) {                                           \
+    Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
+    MICRO_MMA_LOAD_ONE_RHS(peel)                                   \
+    MICRO_MMA_UNROLL(funcl)                                        \
+    MICRO_MMA_WORK(funcw, type, peel)                              \
+  }
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type)                                                  \
+  type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 0)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 1)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 2)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 3)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 4)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 5)                                                            \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 6) MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 7)
+#else
+#define MICRO_MMA_LOAD_TWO_RHS(peel1, right)                                                      \
+  ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr##right + (accRows * peel1)), prhsV##peel1); \
+  __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsV##right[peel1]), &prhsV##peel1);
+
+#define MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, peel1, peel2)           \
+  if (PEEL_MMA > peel2) {                                                                  \
+    PacketBlock<Packet, 2> lhsV20, lhsV21, lhsV22, lhsV23, lhsV24, lhsV25, lhsV26, lhsV27; \
+    __vector_pair plhsV0, plhsV1, plhsV2, plhsV3, plhsV4, plhsV5, plhsV6, plhsV7;          \
+    if (sizeof(type) == 16) {                                                              \
+      MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_TWO_RHS, peel1)                                 \
+    } else {                                                                               \
+      EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                                 \
+      MICRO_MMA_LOAD_ONE_RHS(peel1)                                                        \
+      MICRO_MMA_LOAD_ONE_RHS(peel2)                                                        \
+    }                                                                                      \
+    MICRO_MMA_UNROLL(funcl2)                                                               \
+    MICRO_MMA_WORK(funcw2, type, peel1)                                                    \
+    MICRO_MMA_WORK(funcw2, type, peel2)                                                    \
+  } else {                                                                                 \
+    EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                                   \
+    MICRO_MMA_TYPE_PEEL(funcw1, funcl1, type, peel1)                                       \
+  }
+
+#define MICRO_MMA_UNROLL_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type)                               \
+  type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \
+  __vector_pair prhsV0, prhsV2, prhsV4, prhsV6;                                                         \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 0, 1)                                      \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 2, 3)                                      \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 4, 5)                                      \
+  MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 6, 7)
+#endif
+
+#define MICRO_MMA_UNROLL_TYPE_ONE(funcw, funcl, type) \
+  type rhsV0[1], rhsV1[1], rhsV2[1], rhsV3[1];        \
+  MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 0)
+
+#define MICRO_MMA_UPDATE_RHS1(size, right) rhs_ptr##right += (accRows * size);
+
+#define MICRO_MMA_UPDATE_RHS(size) MICRO_MMA_UNROLL_ITER(MICRO_MMA_UPDATE_RHS1, size)
+
+#define MICRO_MMA_UNROLL_TYPE(MICRO_MMA_TYPE, size)             \
+  MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, RhsPacket) \
+  MICRO_MMA_UPDATE_RHS(size)
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_PEEL, PEEL_MMA)
+#else
+#define MICRO_MMA_UNROLL_TYPE2(MICRO_MMA_TYPE, size)                                                    \
+  MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, MICRO_MMA_WORK_TWO, MICRO_MMA_LOAD_TWO, RhsPacket) \
+  MICRO_MMA_UPDATE_RHS(size)
+
+#define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE2(MICRO_MMA_UNROLL_TYPE_PEEL2, PEEL_MMA)
+#endif
+
+#define MICRO_MMA_ONE MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_ONE, 1)
+
+#define MICRO_MMA_DST_PTR_ONE(iter)       \
+  if (unroll_factor * accItr > iter) {    \
+    bsetzeroMMA(&accZero##iter);          \
+  } else {                                \
+    EIGEN_UNUSED_VARIABLE(accZero##iter); \
+  }
+
+#define MICRO_MMA_DST_PTR MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE)
+
+#define MICRO_MMA_SRC_PTR MICRO_MMA_UNROLL(MICRO_SRC_PTR_ONE)
+
+#define MICRO_MMA_PREFETCH MICRO_MMA_UNROLL(MICRO_PREFETCH_ONE)
+
+#define MICRO_MMA_STORE_ONE(iter, left, right)                                                                 \
+  if (unroll_factor > left) {                                                                                  \
+    storeAccumulator<DataMapper, Packet, MICRO_NORMAL_PARTIAL(left)>(row + left * accCols, res##right, pAlpha, \
+                                                                     accCols2, &accZero##iter);                \
+  }
+
+#define MICRO_MMA_ITER_UNROLL(func)                                                                                 \
+  if (accItr == 1) {                                                                                                \
+    func(0, 0, 0) func(1, 1, 0) func(2, 2, 0) func(3, 3, 0) func(4, 4, 0) func(5, 5, 0) func(6, 6, 0) func(7, 7, 0) \
+  } else if (accItr == 2) {                                                                                         \
+    func(0, 0, 0) func(1, 0, 1) func(2, 1, 0) func(3, 1, 1) func(4, 2, 0) func(5, 2, 1) func(6, 3, 0) func(7, 3, 1) \
+  } else {                                                                                                          \
+    func(0, 0, 0) func(1, 0, 1) func(2, 0, 2) func(3, 0, 3) func(4, 1, 0) func(5, 1, 1) func(6, 1, 2) func(7, 1, 3) \
+  }
+
+#define MICRO_MMA_STORE MICRO_MMA_ITER_UNROLL(MICRO_MMA_STORE_ONE)
+
+#define MICRO_MMA_EXTRA_ROWS(right)                                                                           \
+  gemm_extra_row<Scalar, Packet, DataMapper, accRows, accCols>(                                               \
+      res3##right, blockA, rhs_base + right * accRows * strideB, depth, strideA, offsetA, strideB, row, rows, \
+      remaining_rows, pAlpha, pMask);
+
+#define MICRO_MMA_EXTRA_ROWS1(val, right) MICRO_MMA_EXTRA_ROWS(right);
+
+template <int unroll_factor, typename Scalar, typename Packet, typename RhsPacket, typename DataMapper,
+          const Index accRows, const Index accCols, bool full, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemm_unrolled_MMA_iteration(const DataMapper& res0, const DataMapper& res1,
+                                                     const DataMapper& res2, const DataMapper& res3,
+                                                     const Scalar* lhs_base, const Scalar* rhs_base, Index depth,
+                                                     Index strideA, Index strideB, Index offsetA, Index& row,
+                                                     const Packet& pAlpha, Index accCols2) {
+  const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 = NULL, *rhs_ptr2 = NULL, *rhs_ptr3 = NULL;
+  const Scalar *lhs_ptr0 = NULL, *lhs_ptr1 = NULL, *lhs_ptr2 = NULL, *lhs_ptr3 = NULL, *lhs_ptr4 = NULL,
+               *lhs_ptr5 = NULL, *lhs_ptr6 = NULL, *lhs_ptr7 = NULL;
+  __vector_quad accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;
+
+  if (accItr > 1) {
+    rhs_ptr1 = rhs_base + (accRows * strideB);
+  } else {
+    EIGEN_UNUSED_VARIABLE(strideB);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr1);
+    EIGEN_UNUSED_VARIABLE(res1);
+  }
+  if (accItr > 2) {
+    rhs_ptr2 = rhs_base + (2 * accRows * strideB);
+    rhs_ptr3 = rhs_base + (3 * accRows * strideB);
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr2);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr3);
+    EIGEN_UNUSED_VARIABLE(res2);
+    EIGEN_UNUSED_VARIABLE(res3);
+  }
+
+  MICRO_MMA_SRC_PTR
+  MICRO_MMA_DST_PTR
+
+  Index k = 0, depth2 = depth - PEEL_MMA;
+  for (; k <= depth2; k += PEEL_MMA) {
+    EIGEN_POWER_PREFETCH(rhs_ptr);
+    MICRO_MMA_PREFETCH
+    MICRO_MMA_ONE_PEEL
+  }
+  for (; k < depth; k++) {
+    MICRO_MMA_ONE
+  }
+  MICRO_MMA_STORE
+
+  MICRO_UPDATE
+}
+
+#define MICRO_MMA_UNROLL_ITER2(N, M)                                                                                 \
+  gemm_unrolled_MMA_iteration<N + (M ? 1 : 0), Scalar, Packet, RhsPacket, DataMapper, accRows, accCols, !M, accItr>( \
+      res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, strideB, offsetA, row, pAlpha,                 \
+      M ? remaining_rows : accCols);                                                                                 \
+  if (M) return;
+
+#define MICRO_MMA_ROWS(n)             \
+  while (row + n * accCols <= rows) { \
+    MICRO_MMA_UNROLL_ITER2(n, 0);     \
+  }
+
+template <typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows,
+          const Index accCols, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemmMMA_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index depth,
+                                      Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows,
+                                      Index remaining_rows, const Packet& pAlpha, const Packet& pMask) {
+  const DataMapper res30 = res.getSubMapper(0, col);
+  const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows * 1) : res30;
+  const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows * 2) : res30;
+  const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows * 3) : res30;
+
+  const Scalar* rhs_base = blockB + col * strideB + accRows * offsetB;
+  const Scalar* lhs_base = blockA + accCols * offsetA;
+  Index row = 0;
+
+#define MAX_MMA_UNROLL 7
+
+#if MAX_MMA_UNROLL < 2
+  if (1) {
+#elif MAX_MMA_UNROLL < 4
+  if (accItr <= 2) {
+#else
+  if (accItr == 1) {
+#endif
+    MICRO_MMA_ROWS(MAX_MMA_UNROLL);
+  } else if (accItr == 2) {
+    MICRO_MMA_ROWS(4);
+  } else {
+    MICRO_MMA_ROWS(2);
+  }
+  switch ((rows - row) / accCols) {
+#if MAX_MMA_UNROLL > 7
+    case 7:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 7)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 6
+    case 6:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 6)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 5
+    case 5:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 5)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 4
+    case 4:
+      if (accItr == 1) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 4)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 3
+    case 3:
+      if (accItr <= 2) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 3)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 2
+    case 2:
+      if (accItr <= 2) {
+        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 2)
+      }
+      break;
+#endif
+#if MAX_MMA_UNROLL > 1
+    case 1:
+      MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 1)
+      break;
+#endif
+    default:
+      break;
+  }
+#undef MAX_MMA_UNROLL
+
+  if (remaining_rows > 0) {
+    MICRO_MMA_UNROLL_ITER(MICRO_MMA_EXTRA_ROWS1, 0)
+  }
+}
+
+#define MICRO_MMA_COLS(n)                                                                                          \
+  for (; col + n * accRows <= cols; col += n * accRows) {                                                          \
+    gemmMMA_cols<Scalar, Packet, RhsPacket2, DataMapper, accRows, accCols, n>(                                     \
+        res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlpha, pMask); \
+  }
+
+template <typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows,
+          const Index accCols>
+void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols,
+             Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
+  const Index remaining_rows = rows % accCols;
+
+  if (strideA == -1) strideA = depth;
+  if (strideB == -1) strideB = depth;
+
+  const Packet pAlpha = pset1<Packet>(alpha);
+  const Packet pMask = bmask<Packet>(remaining_rows);
+
+  typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2;
+
+  Index col = 0;
+#ifdef GEMM_MULTIPLE_COLS
+  MICRO_MMA_COLS(4);
+  MICRO_MMA_COLS(2);
+#endif
+  MICRO_MMA_COLS(1);
+
+  if (col != cols) {
+    gemm_extra_cols<Scalar, Packet, DataMapper, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB,
+                                                         col, rows, cols, remaining_rows, pAlpha, pMask);
+  }
+}
+
+#define advanceRows ((LhsIsReal) ? 1 : 2)
+#define advanceCols ((RhsIsReal) ? 1 : 2)
+
+// PEEL_COMPLEX_MMA loop factor.
+#ifdef GEMM_MULTIPLE_COLS
+#define PEEL_COMPLEX_MMA 4
+#else
+#define PEEL_COMPLEX_MMA 3
+#endif
+
+#define MICRO_COMPLEX_MMA_UNROLL(func) func(0) func(1) func(2) func(3)
+
+#define MICRO_COMPLEX_MMA_WORK(func, type, peel)                                                            \
+  if (accItr == 1) {                                                                                        \
+    func(0, type, peel, 0, 0) func(1, type, peel, 1, 0) func(2, type, peel, 2, 0) func(3, type, peel, 3, 0) \
+  } else if (accItr == 2) {                                                                                 \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 1, 0) func(3, type, peel, 1, 1) \
+  } else {                                                                                                  \
+    func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 0, 2) func(3, type, peel, 0, 3) \
+  }
+
+#define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel, left, right)                                        \
+  if (unroll_factor > left) {                                                                            \
+    pgercMMA<Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(                            \
+        &accReal##iter, &accImag##iter, lhsV##left, lhsVi##left, rhsV##right[peel], rhsVi##right[peel]); \
+  }
+
+#ifdef VECTOR_PAIR_LOADS_LHS
+#define MICRO_COMPLEX_MMA_WORK_TWO(iter, type, peel, left, right)                                    \
+  if (unroll_factor > left) {                                                                        \
+    pgercMMA<Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(                        \
+        &accReal##iter, &accImag##iter, lhsV2##left.packet[peel & 1], lhsVi2##left.packet[peel & 1], \
+        rhsV##right[peel], rhsVi##right[peel]);                                                      \
+  }
+
+#define MICRO_COMPLEX_MMA_LOAD1_TWO(lhs_ptr, left)                                                  \
+  if (!LhsIsReal && (unroll_factor > left)) {                                                       \
+    if (MICRO_NORMAL(left)) {                                                                       \
+      ploadLhsMMA(reinterpret_cast<const double*>(lhs_ptr_real##left + imag_delta), plhsVi##left);  \
+      __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&lhsVi2##left.packet), &plhsVi##left); \
+    } else {                                                                                        \
+      lhsVi2##left.packet[0] = ploadLhs<Packet>(lhs_ptr_real##left + imag_delta2);                  \
+      lhsVi2##left.packet[1] = ploadLhs<Packet>(lhs_ptr_real##left + imag_delta2 + accCols2);       \
+      EIGEN_UNUSED_VARIABLE(plhsVi##left);                                                          \
+    }                                                                                               \
+  } else {                                                                                          \
+    EIGEN_UNUSED_VARIABLE(lhsVi2##left);                                                            \
+    EIGEN_UNUSED_VARIABLE(plhsVi##left);                                                            \
+  }                                                                                                 \
+  MICRO_MMA_LOAD1_TWO(lhs_ptr_real, left)
+
+#define MICRO_COMPLEX_MMA_LOAD_TWO(left) MICRO_COMPLEX_MMA_LOAD1_TWO(lhs_ptr, left)
+#endif
+
+#define MICRO_COMPLEX_MMA_LOAD_RHS1(peel, right)                             \
+  ploadRhsMMA(rhs_ptr_real##right + (accRows * peel), rhsV##right[peel]);    \
+  if (!RhsIsReal) {                                                          \
+    ploadRhsMMA(rhs_ptr_imag##right + (accRows * peel), rhsVi##right[peel]); \
+  }
+
+#define MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel) MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_RHS1, peel)
+
+#define MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, peel) \
+  if (PEEL_COMPLEX_MMA > peel) {                              \
+    Packet lhsV0, lhsV1, lhsV2, lhsV3;                        \
+    Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3;                    \
+    MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel)                      \
+    MICRO_COMPLEX_MMA_UNROLL(funcl)                           \
+    MICRO_COMPLEX_MMA_WORK(funcw, type, peel)                 \
+  }
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type)                                                      \
+  type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], \
+      rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1];                      \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 0)                                                                \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 1)                                                                \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 2) MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 3)
+#else
+#define MICRO_COMPLEX_MMA_LOAD_TWO_RHS(peel1, right)                                                      \
+  ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr_real##right + (accRows * peel1)), prhsV##peel1);    \
+  __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsV##right[peel1]), &prhsV##peel1);            \
+  if (!RhsIsReal) {                                                                                       \
+    ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr_imag##right + (accRows * peel1)), prhsVi##peel1); \
+    __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsVi##right[peel1]), &prhsVi##peel1);        \
+  } else {                                                                                                \
+    EIGEN_UNUSED_VARIABLE(prhsVi##peel1);                                                                 \
+  }
+
+#define MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, peel1, peel2) \
+  if (PEEL_COMPLEX_MMA > peel2) {                                                        \
+    PacketBlock<Packet, 2> lhsV20, lhsV21, lhsV22, lhsV23;                               \
+    PacketBlock<Packet, 2> lhsVi20, lhsVi21, lhsVi22, lhsVi23;                           \
+    __vector_pair plhsV0, plhsV1, plhsV2, plhsV3;                                        \
+    __vector_pair plhsVi0, plhsVi1, plhsVi2, plhsVi3;                                    \
+    if (sizeof(type) == 16) {                                                            \
+      MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_TWO_RHS, peel1)                       \
+    } else {                                                                             \
+      EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                               \
+      EIGEN_UNUSED_VARIABLE(prhsVi##peel1);                                              \
+      MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel1);                                             \
+      MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel2);                                             \
+    }                                                                                    \
+    MICRO_COMPLEX_MMA_UNROLL(funcl2)                                                     \
+    MICRO_COMPLEX_MMA_WORK(funcw2, type, peel1)                                          \
+    MICRO_COMPLEX_MMA_WORK(funcw2, type, peel2)                                          \
+  } else {                                                                               \
+    EIGEN_UNUSED_VARIABLE(prhsV##peel1);                                                 \
+    EIGEN_UNUSED_VARIABLE(prhsVi##peel1);                                                \
+    MICRO_COMPLEX_MMA_TYPE_PEEL(funcw1, funcl1, type, peel1)                             \
+  }
+
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type)                                   \
+  type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], \
+      rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1];                      \
+  __vector_pair prhsV0, prhsV2;                                                                                     \
+  __vector_pair prhsVi0, prhsVi2;                                                                                   \
+  MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 0, 1)                                          \
+  MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, 2, 3)
+#endif
+
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(funcw, funcl, type)                              \
+  type rhsV0[1], rhsVi0[1], rhsV1[1], rhsVi1[1], rhsV2[1], rhsVi2[1], rhsV3[1], rhsVi3[1]; \
+  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 0)
+
+#define MICRO_COMPLEX_MMA_UPDATE_RHS1(size, right) \
+  rhs_ptr_real##right += (accRows * size);         \
+  if (!RhsIsReal) rhs_ptr_imag##right += (accRows * size);
+
+#define MICRO_COMPLEX_MMA_UPDATE_RHS(size) MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_UPDATE_RHS1, size)
+
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_TYPE, size)                     \
+  MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, RhsPacket) \
+  MICRO_COMPLEX_MMA_UPDATE_RHS(size);
+
+#ifndef VECTOR_PAIR_LOADS_LHS
+#define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL, PEEL_COMPLEX_MMA)
+#else
+#define MICRO_COMPLEX_MMA_UNROLL_TYPE2(MICRO_COMPLEX_MMA_TYPE, size)                                     \
+  MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, MICRO_COMPLEX_MMA_WORK_TWO, \
+                         MICRO_COMPLEX_MMA_LOAD_TWO, RhsPacket)                                          \
+  MICRO_COMPLEX_MMA_UPDATE_RHS(size);
+
+#define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE2(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL2, PEEL_COMPLEX_MMA)
+#endif
+
+#define MICRO_COMPLEX_MMA_ONE MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE, 1)
+
+#define MICRO_COMPLEX_MMA_DST_PTR_ONE(iter) \
+  if (unroll_factor * accItr > iter) {      \
+    bsetzeroMMA(&accReal##iter);            \
+    bsetzeroMMA(&accImag##iter);            \
+  } else {                                  \
+    EIGEN_UNUSED_VARIABLE(accReal##iter);   \
+    EIGEN_UNUSED_VARIABLE(accImag##iter);   \
+  }
+
+#define MICRO_COMPLEX_MMA_DST_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE)
+
+#define MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE)
+
+#define MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_PREFETCH_ONE)
+
+#define MICRO_COMPLEX_MMA_STORE_ONE(iter, left, right)                                                                 \
+  if (unroll_factor > left) {                                                                                          \
+    storeComplexAccumulator<DataMapper, Packet, Packetc, accCols, (unroll_factor != (left + 1)) ? accCols : accCols2>( \
+        row + left * accCols, res##right, pAlphaReal, pAlphaImag, pMask, &accReal##iter, &accImag##iter);              \
+  }
+
+#define MICRO_COMPLEX_MMA_ITER_UNROLL(func)                 \
+  if (accItr == 1) {                                        \
+    func(0, 0, 0) func(1, 1, 0) func(2, 2, 0) func(3, 3, 0) \
+  } else if (accItr == 2) {                                 \
+    func(0, 0, 0) func(1, 0, 1) func(2, 1, 0) func(3, 1, 1) \
+  } else {                                                  \
+    func(0, 0, 0) func(1, 0, 1) func(2, 0, 2) func(3, 0, 3) \
+  }
+
+#define MICRO_COMPLEX_MMA_STORE MICRO_COMPLEX_MMA_ITER_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE)
+
+#define MICRO_COMPLEX_MMA_EXTRA_ROWS(right)                                                                            \
+  gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, \
+                         RhsIsReal>(res3##right, blockA, rhs_base + right * accRows * (RhsIsReal ? 1 : 2) * strideB,   \
+                                    depth, strideA, offsetA, strideB, row, rows, remaining_rows, pAlphaReal,           \
+                                    pAlphaImag, pMask);
+
+#define MICRO_COMPLEX_MMA_EXTRA_ROWS1(val, right) MICRO_COMPLEX_MMA_EXTRA_ROWS(right);
+
+template <int unroll_factor, typename Scalar, typename Packet, typename Packetc, typename RhsPacket,
+          typename DataMapper, const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs,
+          bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_MMA_iteration(const DataMapper& res0, const DataMapper& res1,
+                                                             const DataMapper& res2, const DataMapper& res3,
+                                                             const Scalar* lhs_base, const Scalar* rhs_base,
+                                                             Index depth, Index strideA, Index offsetA, Index strideB,
+                                                             Index& row, const Packet& pAlphaReal,
+                                                             const Packet& pAlphaImag, const Packet& pMask) {
+  const Scalar *rhs_ptr_real0 = rhs_base, *rhs_ptr_real1 = NULL, *rhs_ptr_real2 = NULL, *rhs_ptr_real3 = NULL;
+  const Scalar *rhs_ptr_imag0 = NULL, *rhs_ptr_imag1 = NULL, *rhs_ptr_imag2 = NULL, *rhs_ptr_imag3 = NULL;
+  const Index imag_delta = accCols * strideA;
+  const Index imag_delta2 = accCols2 * strideA;
+
+  if (!RhsIsReal) {
+    rhs_ptr_imag0 = rhs_base + accRows * strideB;
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag0);
+  }
+  if (accItr > 1) {
+    if (!RhsIsReal) {
+      rhs_ptr_real1 = rhs_base + (2 * accRows * strideB);
+      rhs_ptr_imag1 = rhs_base + (3 * accRows * strideB);
+    } else {
+      rhs_ptr_real1 = rhs_base + accRows * strideB;
+      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1);
+    }
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_real1);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1);
+    EIGEN_UNUSED_VARIABLE(res1);
+  }
+  if (accItr > 2) {
+    if (!RhsIsReal) {
+      rhs_ptr_real2 = rhs_base + (4 * accRows * strideB);
+      rhs_ptr_imag2 = rhs_base + (5 * accRows * strideB);
+      rhs_ptr_real3 = rhs_base + (6 * accRows * strideB);
+      rhs_ptr_imag3 = rhs_base + (7 * accRows * strideB);
+    } else {
+      rhs_ptr_real2 = rhs_base + (2 * accRows * strideB);
+      rhs_ptr_real3 = rhs_base + (3 * accRows * strideB);
+      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2);
+      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3);
+    }
+  } else {
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_real2);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_real3);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2);
+    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3);
+    EIGEN_UNUSED_VARIABLE(res2);
+    EIGEN_UNUSED_VARIABLE(res3);
+  }
+  const Scalar *lhs_ptr_real0 = NULL, *lhs_ptr_real1 = NULL;
+  const Scalar *lhs_ptr_real2 = NULL, *lhs_ptr_real3 = NULL;
+  __vector_quad accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3;
+
+  MICRO_COMPLEX_MMA_SRC_PTR
+  MICRO_COMPLEX_MMA_DST_PTR
+
+  Index k = 0, depth2 = depth - PEEL_COMPLEX_MMA;
+  for (; k <= depth2; k += PEEL_COMPLEX_MMA) {
+    EIGEN_POWER_PREFETCH(rhs_ptr_real);
+    if (!RhsIsReal) {
+      EIGEN_POWER_PREFETCH(rhs_ptr_imag);
+    }
+    MICRO_COMPLEX_MMA_PREFETCH
+    MICRO_COMPLEX_MMA_ONE_PEEL
+  }
+  for (; k < depth; k++) {
+    MICRO_COMPLEX_MMA_ONE
+  }
+  MICRO_COMPLEX_MMA_STORE
+
+  MICRO_COMPLEX_UPDATE
+}
+
+#define MICRO_COMPLEX_MMA_UNROLL_ITER2(N, M)                                                                           \
+  gemm_complex_unrolled_MMA_iteration<N + (M ? 1 : 0), Scalar, Packet, Packetc, RhsPacket, DataMapper, accRows,        \
+                                      accCols, M ? M : accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal,      \
+                                      accItr>(res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, offsetA, \
+                                              strideB, row, pAlphaReal, pAlphaImag, pMask);                            \
+  if (M) return;
+
+#define MICRO_COMPLEX_MMA_ROWS(n)         \
+  while (row + n * accCols <= rows) {     \
+    MICRO_COMPLEX_MMA_UNROLL_ITER2(n, 0); \
+  }
+
+template <typename Scalar, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper,
+          const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal,
+          bool RhsIsReal, const Index accItr>
+EIGEN_ALWAYS_INLINE void gemmMMA_complex_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB,
+                                              Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB,
+                                              Index col, Index rows, Index remaining_rows, const Packet& pAlphaReal,
+                                              const Packet& pAlphaImag, const Packet& pMask) {
+  const DataMapper res30 = res.getSubMapper(0, col);
+  const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows * 1) : res30;
+  const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows * 2) : res30;
+  const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows * 3) : res30;
+
+  const Scalar* rhs_base = blockB + advanceCols * col * strideB + accRows * offsetB;
+  const Scalar* lhs_base = blockA + accCols * offsetA;
+  Index row = 0;
+
+#define MAX_COMPLEX_MMA_UNROLL 4
+
+#if MAX_COMPLEX_MMA_UNROLL < 2
+  if (1) {
+#elif MAX_COMPLEX_MMA_UNROLL < 4
+  if (accItr <= 2) {
+#else
+  if (accItr == 1) {
+#endif
+    MICRO_COMPLEX_MMA_ROWS(MAX_COMPLEX_MMA_UNROLL);
+  } else if (accItr == 2) {
+    MICRO_COMPLEX_MMA_ROWS(2);
+  } else {
+    MICRO_COMPLEX_MMA_ROWS(1);
+  }
+  switch ((rows - row) / accCols) {
+#if MAX_COMPLEX_MMA_UNROLL > 3
+    case 3:
+      if (accItr == 1) {
+        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 3)
+      }
+      break;
+#endif
+#if MAX_COMPLEX_MMA_UNROLL > 2
+    case 2:
+      if (accItr == 1) {
+        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 2)
+      }
+      break;
+#endif
+#if MAX_COMPLEX_MMA_UNROLL > 1
+    case 1:
+      if (accItr <= 2) {
+        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 1)
+      }
+      break;
+#endif
+    default:
+      break;
+  }
+#undef MAX_COMPLEX_MMA_UNROLL
+
+  if (remaining_rows > 0) {
+    MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_EXTRA_ROWS1, 0)
+  }
+}
+
+#define MICRO_COMPLEX_MMA_COLS(n)                                                                                      \
+  for (; col + n * accRows <= cols; col += n * accRows) {                                                              \
+    gemmMMA_complex_cols<Scalar, Packet, Packetc, RhsPacket2, DataMapper, accRows, accCols, ConjugateLhs,              \
+                         ConjugateRhs, LhsIsReal, RhsIsReal, n>(res, blockA, blockB, depth, strideA, offsetA, strideB, \
+                                                                offsetB, col, rows, remaining_rows, pAlphaReal,        \
+                                                                pAlphaImag, pMask);                                    \
+  }
+
+template <typename LhsScalar, typename RhsScalar, typename Scalarc, typename Scalar, typename Packet, typename Packetc,
+          typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, bool ConjugateLhs,
+          bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
+void gemm_complexMMA(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc, Index rows, Index depth,
+                     Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
+  const Index remaining_rows = rows % accCols;
+
+  if (strideA == -1) strideA = depth;
+  if (strideB == -1) strideB = depth;
+
+  const Packet pAlphaReal = pset1<Packet>(alpha.real());
+  const Packet pAlphaImag = pset1<Packet>(alpha.imag());
+  const Packet pMask = bmask<Packet>(remaining_rows);
+
+  const Scalar* blockA = (Scalar*)blockAc;
+  const Scalar* blockB = (Scalar*)blockBc;
+
+  typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2;
+
+  Index col = 0;
+#ifdef GEMM_MULTIPLE_COLS
+  MICRO_COMPLEX_MMA_COLS(4);
+  MICRO_COMPLEX_MMA_COLS(2);
+#endif
+  MICRO_COMPLEX_MMA_COLS(1);
+
+  if (col != cols) {
+    gemm_complex_extra_cols<Scalar, Packet, Packetc, DataMapper, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
+                            RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols,
+                                       remaining_rows, pAlphaReal, pAlphaImag, pMask);
+  }
+}
+
+#undef accColsC
+#undef advanceRows
+#undef advanceCols
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#if defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
+#pragma GCC pop_options
+#endif
+
+#endif  // EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H

include/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h

@@ -0,0 +1,742 @@
+#ifndef EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H
+#define EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H
+
+#if EIGEN_COMP_LLVM
+#define BFLOAT16_UNROLL _Pragma("unroll 8")
+#else
+#define BFLOAT16_UNROLL _Pragma("GCC unroll(8)")
+#endif
+
+namespace Eigen {
+
+namespace internal {
+
+template <bool zero>
+EIGEN_ALWAYS_INLINE Packet8bf loadBfloat16(const bfloat16* indexA) {
+  Packet8bf lhs1 = ploadu<Packet8bf>(indexA);
+  if (zero) {
+    Packet8bf lhs2 = pset1<Packet8bf>(Eigen::bfloat16(0));
+    return vec_mergeh(lhs1.m_val, lhs2.m_val);
+  } else {
+    return lhs1;
+  }
+}
+
+template <bool zero>
+EIGEN_ALWAYS_INLINE Packet8bf loadRhsBfloat16(const bfloat16* blockB, Index strideB, Index i) {
+  return loadBfloat16<zero>(blockB + strideB * i);
+}
+
+template <Index num_acc, Index num_packets, bool zero, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs,
+          Index num_lhs>
+EIGEN_ALWAYS_INLINE void KLoop(const bfloat16* indexA, const bfloat16* indexB, __vector_quad (&quad_acc)[num_acc],
+                               Index strideB, Index k, Index offsetB, Index extra_cols, Index extra_rows) {
+  Packet8bf lhs[num_lhs], rhs[num_rhs];
+
+  BFLOAT16_UNROLL
+  for (Index i = 0; i < (num_rhs - (rhsExtraCols ? 1 : 0)); i++) {
+    rhs[i] = loadRhsBfloat16<zero>(indexB + k * 4, strideB, i);
+  }
+  if (rhsExtraCols) {
+    rhs[num_rhs - 1] = loadRhsBfloat16<zero>(indexB + k * extra_cols - offsetB, strideB, num_rhs - 1);
+  }
+
+  indexA += k * (lhsExtraRows ? extra_rows : num_packets);
+  if (num_lhs == 1) {
+    lhs[0] = loadBfloat16<zero>(indexA);
+  } else {
+    BFLOAT16_UNROLL
+    for (Index j = 0; j < num_lhs; j += 2) {
+      Packet8bf lhs1 = ploadu<Packet8bf>(indexA + (j + 0) * (zero ? 4 : 8));
+      if (zero) {
+        Packet8bf lhs2 = pset1<Packet8bf>(Eigen::bfloat16(0));
+        lhs[j + 0] = vec_mergeh(lhs1.m_val, lhs2.m_val);
+        lhs[j + 1] = vec_mergel(lhs1.m_val, lhs2.m_val);
+      } else {
+        lhs[j + 0] = lhs1;
+        lhs[j + 1] = ploadu<Packet8bf>(indexA + (j + 1) * 8);
+      }
+    }
+  }
+
+  BFLOAT16_UNROLL
+  for (Index i = 0, x = 0; i < num_rhs; i++) {
+    BFLOAT16_UNROLL
+    for (Index j = 0; j < num_lhs; j++, x++) {
+      __builtin_mma_xvbf16ger2pp(&(quad_acc[x]), reinterpret_cast<Packet16uc>(rhs[i].m_val),
+                                 reinterpret_cast<Packet16uc>(lhs[j].m_val));
+    }
+  }
+}
+
+template <Index num_acc>
+EIGEN_ALWAYS_INLINE void zeroAccumulators(__vector_quad (&quad_acc)[num_acc]) {
+  BFLOAT16_UNROLL
+  for (Index k = 0; k < num_acc; k++) __builtin_mma_xxsetaccz(&(quad_acc[k]));
+}
+
+template <Index num_acc>
+EIGEN_ALWAYS_INLINE void disassembleAccumulators(__vector_quad (&quad_acc)[num_acc], Packet4f (&acc)[num_acc][4]) {
+  BFLOAT16_UNROLL
+  for (Index k = 0; k < num_acc; k++) __builtin_mma_disassemble_acc((void*)acc[k], &(quad_acc[k]));
+}
+
+template <Index num_acc, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs, Index num_lhs>
+EIGEN_ALWAYS_INLINE void outputResults(Packet4f (&acc)[num_acc][4], Index rows, const Packet4f pAlpha, float* result,
+                                       const Index extra_cols, Index extra_rows) {
+  BFLOAT16_UNROLL
+  for (Index i = 0, k = 0; i < num_rhs - (rhsExtraCols ? 1 : 0); i++, result += 4 * rows) {
+    BFLOAT16_UNROLL
+    for (Index j = 0; j < num_lhs; j++, k++) {
+      storeResults<false, lhsExtraRows>(acc[k], rows, pAlpha, result + j * 4, extra_cols, extra_rows);
+    }
+  }
+  if (rhsExtraCols) {
+    storeResults<rhsExtraCols, lhsExtraRows>(acc[num_acc - 1], rows, pAlpha, result, extra_cols, extra_rows);
+  }
+}
+
+template <const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows, bool multiIter = false>
+EIGEN_ALWAYS_INLINE void colLoopBodyIter(Index depth, Index rows, const Packet4f pAlpha, const bfloat16* indexA,
+                                         const bfloat16* indexB, Index strideB, Index offsetB, float* result,
+                                         const Index extra_cols, const Index extra_rows) {
+  constexpr Index num_lhs = multiIter ? (num_packets / 4) : 1;
+  constexpr Index num_rhs = (num_acc + num_lhs - 1) / num_lhs;
+
+  for (Index offset_row = 0; offset_row < num_packets; offset_row += 4, indexA += (multiIter ? 0 : 8),
+             indexB += (multiIter ? (num_rhs * strideB) : 0), result += (multiIter ? (4 * rows * num_rhs) : 4)) {
+    Packet4f acc[num_acc][4];
+    __vector_quad quad_acc[num_acc];
+
+    zeroAccumulators<num_acc>(quad_acc);
+
+    Index k;
+    for (k = 0; k + 2 <= depth; k += 2) {
+      KLoop<num_acc, num_packets, false, rhsExtraCols, lhsExtraRows, num_rhs, num_lhs>(
+          indexA, indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows);
+    }
+    if (depth & 1) {
+      KLoop<num_acc, num_packets, true, rhsExtraCols, lhsExtraRows, num_rhs, num_lhs>(
+          indexA - (multiIter ? 0 : offset_row), indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows);
+    }
+
+    disassembleAccumulators<num_acc>(quad_acc, acc);
+
+    outputResults<num_acc, rhsExtraCols, lhsExtraRows, num_rhs, num_lhs>(acc, rows, pAlpha, result, extra_cols,
+                                                                         extra_rows);
+  }
+}
+
+#define MAX_BFLOAT16_ACC 8
+
+template <const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>
+void colLoopBody(Index& col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA,
+                 const bfloat16* indexB, Index strideB, Index offsetB, float* result) {
+  constexpr Index step = (num_acc * 4);  // each accumulator has 4 elements
+  const Index extra_cols = (rhsExtraCols) ? (cols & 3) : 0;
+  const Index extra_rows = (lhsExtraRows) ? (rows & 3) : 0;
+  constexpr bool multiIters = !rhsExtraCols && (num_acc == MAX_BFLOAT16_ACC);
+  constexpr bool normIters = multiIters && ((num_acc % (num_packets / 4)) == 0);
+
+  do {
+    colLoopBodyIter<num_acc, num_packets, rhsExtraCols, lhsExtraRows, normIters>(
+        depth, rows, pAlpha, indexA, indexB, strideB, offsetB, result, extra_cols, extra_rows);
+
+    indexB += strideB * num_acc;
+    result += rows * step;
+  } while (multiIters && (step <= cols - (col += step)));
+}
+
+template <const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>
+EIGEN_ALWAYS_INLINE void colLoopBodyExtraN(Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha,
+                                           const bfloat16* indexA, const bfloat16* blockB, Index strideB, Index offsetB,
+                                           float* result) {
+  if (MAX_BFLOAT16_ACC > num_acc) {
+    colLoopBody<num_acc + (rhsExtraCols ? 1 : 0), num_packets, rhsExtraCols, lhsExtraRows>(
+        col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result);
+  }
+}
+
+template <const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>
+void colLoopBodyExtra(Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA,
+                      const bfloat16* blockB, Index strideB, Index offsetB, float* result) {
+  switch ((cols - col) >> 2) {
+    case 7:
+      colLoopBodyExtraN<7, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
+                                                                    strideB, offsetB, result);
+      break;
+    case 6:
+      colLoopBodyExtraN<6, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
+                                                                    strideB, offsetB, result);
+      break;
+    case 5:
+      colLoopBodyExtraN<5, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
+                                                                    strideB, offsetB, result);
+      break;
+    case 4:
+      colLoopBodyExtraN<4, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
+                                                                    strideB, offsetB, result);
+      break;
+    case 3:
+      colLoopBodyExtraN<3, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
+                                                                    strideB, offsetB, result);
+      break;
+    case 2:
+      colLoopBodyExtraN<2, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
+                                                                    strideB, offsetB, result);
+      break;
+    case 1:
+      colLoopBodyExtraN<1, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
+                                                                    strideB, offsetB, result);
+      break;
+    default:
+      if (rhsExtraCols) {
+        colLoopBody<1, num_packets, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
+                                                        offsetB, result);
+      }
+      break;
+  }
+}
+
+template <const Index num_packets, bool lhsExtraRows = false>
+EIGEN_ALWAYS_INLINE void colLoops(Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA,
+                                  const bfloat16* blockB, Index strideB, Index offsetB, float* result) {
+  Index col = 0;
+  if (cols >= (MAX_BFLOAT16_ACC * 4)) {
+    colLoopBody<MAX_BFLOAT16_ACC, num_packets, false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
+                                                                    strideB, 0, result);
+    blockB += (strideB >> 2) * col;
+    result += rows * col;
+  }
+  if (cols & 3) {
+    colLoopBodyExtra<num_packets, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB,
+                                                      result);
+  } else {
+    colLoopBodyExtra<num_packets, false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, 0,
+                                                       result);
+  }
+}
+
+EIGEN_ALWAYS_INLINE Packet8bf convertF32toBF16(const float* res) {
+  Packet16uc fp16[2];
+  __vector_pair fp16_vp = *reinterpret_cast<__vector_pair*>(const_cast<float*>(res));
+  __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(fp16), &fp16_vp);
+  fp16[0] = __builtin_vsx_xvcvspbf16(fp16[0]);
+  fp16[1] = __builtin_vsx_xvcvspbf16(fp16[1]);
+  return vec_pack(reinterpret_cast<Packet4ui>(fp16[0]), reinterpret_cast<Packet4ui>(fp16[1]));
+}
+
+template <typename DataMapper, const Index size>
+EIGEN_ALWAYS_INLINE void convertArrayF32toBF16Col(float* result, Index col, Index rows, const DataMapper& res) {
+  const DataMapper res2 = res.getSubMapper(0, col);
+  Index row;
+  float* result2 = result + col * rows;
+  for (row = 0; row + 8 <= rows; row += 8, result2 += 8) {
+    // get and save block
+    PacketBlock<Packet8bf, size> block;
+    BFLOAT16_UNROLL
+    for (Index j = 0; j < size; j++) {
+      block.packet[j] = convertF32toBF16(result2 + j * rows);
+    }
+    res2.template storePacketBlock<Packet8bf, size>(row, 0, block);
+  }
+  // extra rows
+  if (row < rows) {
+    BFLOAT16_UNROLL
+    for (Index j = 0; j < size; j++) {
+      Packet8bf fp16 = convertF32toBF16(result2 + j * rows);
+      res2.template storePacketPartial<Packet8bf>(row, j, fp16, rows & 7);
+    }
+  }
+}
+
+template <const Index size, bool non_unit_stride = false>
+EIGEN_ALWAYS_INLINE void convertPointerF32toBF16(Index& i, float* result, Index rows, bfloat16*& dst,
+                                                 Index resInc = 1) {
+  constexpr Index extra = ((size < 8) ? 8 : size);
+  while (i + size <= rows) {
+    PacketBlock<Packet8bf, (size + 7) / 8> r32;
+    r32.packet[0] = convertF32toBF16(result + i + 0);
+    if (size >= 16) {
+      r32.packet[1] = convertF32toBF16(result + i + 8);
+    }
+    if (size >= 32) {
+      r32.packet[2] = convertF32toBF16(result + i + 16);
+      r32.packet[3] = convertF32toBF16(result + i + 24);
+    }
+    storeBF16fromResult<size, non_unit_stride, 0>(dst, r32.packet[0], resInc, rows & 7);
+    if (size >= 16) {
+      storeBF16fromResult<size, non_unit_stride, 8>(dst, r32.packet[1], resInc);
+    }
+    if (size >= 32) {
+      storeBF16fromResult<size, non_unit_stride, 16>(dst, r32.packet[2], resInc);
+      storeBF16fromResult<size, non_unit_stride, 24>(dst, r32.packet[3], resInc);
+    }
+    i += extra;
+    dst += extra * resInc;
+    if (size != 32) break;
+  }
+}
+
+template <bool non_unit_stride = false>
+EIGEN_ALWAYS_INLINE void convertArrayPointerF32toBF16(float* result, Index rows, bfloat16* dst, Index resInc = 1) {
+  Index i = 0;
+  convertPointerF32toBF16<32, non_unit_stride>(i, result, rows, dst, resInc);
+  convertPointerF32toBF16<16, non_unit_stride>(i, result, rows, dst, resInc);
+  convertPointerF32toBF16<8, non_unit_stride>(i, result, rows, dst, resInc);
+  convertPointerF32toBF16<1, non_unit_stride>(i, result, rows, dst, resInc);
+}
+
+template <typename DataMapper>
+EIGEN_ALWAYS_INLINE void convertArrayF32toBF16(float* result, Index cols, Index rows, const DataMapper& res) {
+  Index col;
+  for (col = 0; col + 4 <= cols; col += 4) {
+    convertArrayF32toBF16Col<DataMapper, 4>(result, col, rows, res);
+  }
+  // extra cols
+  switch (cols - col) {
+    case 1:
+      convertArrayF32toBF16Col<DataMapper, 1>(result, col, rows, res);
+      break;
+    case 2:
+      convertArrayF32toBF16Col<DataMapper, 2>(result, col, rows, res);
+      break;
+    case 3:
+      convertArrayF32toBF16Col<DataMapper, 3>(result, col, rows, res);
+      break;
+  }
+}
+
+template <Index size>
+EIGEN_ALWAYS_INLINE void calcColLoops(const bfloat16*& indexA, Index& row, Index depth, Index cols, Index rows,
+                                      const Packet4f pAlpha, const bfloat16* indexB, Index strideB, Index offsetA,
+                                      Index offsetB, Index bigSuffix, float* result) {
+  if ((size == 16) || (rows & size)) {
+    indexA += size * offsetA;
+    colLoops<size>(depth, cols, rows, pAlpha, indexA, indexB, strideB, offsetB, result + row);
+    row += size;
+    indexA += bigSuffix * size / 16;
+  }
+}
+
+template <typename DataMapper>
+void gemmMMAbfloat16(const DataMapper& res, const bfloat16* indexA, const bfloat16* indexB, Index rows, Index depth,
+                     Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
+  float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
+  const Packet4f pAlpha = pset1<Packet4f>(falpha);
+  ei_declare_aligned_stack_constructed_variable(float, result, cols* rows, 0);
+
+  convertArrayBF16toF32<DataMapper>(result, cols, rows, res);
+
+  if (strideA == -1) strideA = depth;
+  if (strideB == -1) strideB = depth;
+  // Packing is done in blocks.
+  // There's 4 possible sizes of blocks
+  // Blocks of 8 columns with 16 elements (8x16)
+  // Blocks of 8 columns with 8 elements (8x8). This happens when there's 16 > rows >= 8
+  // Blocks of 8 columns with 4 elements (8x4). This happens when there's 8 > rows >= 4
+  // Blocks of 8 columns with < 4 elements. This happens when there's less than 4 remaining rows
+
+  // Loop for LHS standard block (8x16)
+  Index bigSuffix = (2 * 8) * (strideA - offsetA);
+  indexB += 4 * offsetB;
+  strideB *= 4;
+  offsetB *= 3;
+
+  Index row = 0;
+  while (row + 16 <= rows) {
+    calcColLoops<16>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result);
+  }
+  // LHS (8x8) block
+  calcColLoops<8>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result);
+  // LHS (8x4) block
+  calcColLoops<4>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result);
+  // extra rows
+  if (rows & 3) {
+    // This index is the beginning of remaining block.
+    colLoops<4, true>(depth, cols, rows, pAlpha, indexA, indexB, strideB, offsetB, result + row);
+  }
+
+  // Convert back to bfloat16
+  convertArrayF32toBF16<DataMapper>(result, cols, rows, res);
+}
+
+#undef MAX_BFLOAT16_ACC
+
+#if !EIGEN_ALTIVEC_DISABLE_MMA
+template <Index num_acc, typename LhsMapper, bool zero>
+EIGEN_ALWAYS_INLINE void loadVecLoop(Index k, LhsMapper& lhs, Packet8bf (&a0)[num_acc], Packet8bf b1) {
+  a0[k + 0] = lhs.template loadPacket<Packet8bf>(k * 4, 0);
+  if (!zero) {
+    b1 = lhs.template loadPacket<Packet8bf>(k * 4, 1);
+  }
+  if (num_acc > (k + 1)) {
+    a0[k + 1] = vec_mergel(a0[k + 0].m_val, b1.m_val);
+  }
+  a0[k + 0] = vec_mergeh(a0[k + 0].m_val, b1.m_val);
+}
+
+template <Index num_acc>
+EIGEN_ALWAYS_INLINE void multVec(__vector_quad (&quad_acc)[num_acc], Packet8bf (&a0)[num_acc], Packet8bf b0) {
+  BFLOAT16_UNROLL
+  for (Index k = 0; k < num_acc; k++) {
+    __builtin_mma_xvbf16ger2pp(&(quad_acc[k]), reinterpret_cast<Packet16uc>(b0.m_val),
+                               reinterpret_cast<Packet16uc>(a0[k].m_val));
+  }
+}
+
+template <Index num_acc, typename LhsMapper, typename RhsMapper, bool zero, bool linear>
+EIGEN_ALWAYS_INLINE void vecColLoop(Index j, LhsMapper& lhs, RhsMapper& rhs, __vector_quad (&quad_acc)[num_acc]) {
+  Packet8bf a0[num_acc];
+  Packet8bf b1 = pset1<Packet8bf>(Eigen::bfloat16(0));
+  Packet8bf b0 = loadColData<RhsMapper, linear>(rhs, j);
+
+  if (zero) {
+    b0 = vec_mergeh(b0.m_val, b1.m_val);
+  }
+
+  using LhsSubMapper = typename LhsMapper::SubMapper;
+
+  LhsSubMapper lhs2 = lhs.getSubMapper(0, j);
+  BFLOAT16_UNROLL
+  for (Index k = 0; k < num_acc; k += 2) {
+    loadVecLoop<num_acc, LhsSubMapper, zero>(k, lhs2, a0, b1);
+  }
+
+  multVec<num_acc>(quad_acc, a0, b0);
+}
+
+#define MAX_BFLOAT16_VEC_ACC 8
+
+template <const Index num_acc, typename LhsMapper, typename RhsMapper, bool extraRows, bool linear>
+void colVecColLoopBody(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha,
+                       float* result) {
+  constexpr Index step = (num_acc * 4);
+  const Index extra_rows = (extraRows) ? (rows & 3) : 0;
+  constexpr bool multiIters = !extraRows && (num_acc == MAX_BFLOAT16_VEC_ACC);
+
+  do {
+    Packet4f acc[num_acc][4];
+    __vector_quad quad_acc[num_acc];
+
+    zeroAccumulators<num_acc>(quad_acc);
+
+    using LhsSubMapper = typename LhsMapper::SubMapper;
+
+    LhsSubMapper lhs2 = lhs.getSubMapper(row, 0);
+    for (Index j = 0; j + 2 <= cend; j += 2) {
+      vecColLoop<num_acc, LhsSubMapper, RhsMapper, false, linear>(j, lhs2, rhs, quad_acc);
+    }
+    if (cend & 1) {
+      vecColLoop<num_acc, LhsSubMapper, RhsMapper, true, linear>(cend - 1, lhs2, rhs, quad_acc);
+    }
+
+    disassembleAccumulators<num_acc>(quad_acc, acc);
+
+    outputVecColResults<num_acc, extraRows>(acc, result, pAlpha, extra_rows);
+
+    result += step;
+  } while (multiIters && (step <= rows - (row += step)));
+}
+
+template <const Index num_acc, typename LhsMapper, typename RhsMapper, bool extraRows, bool linear>
+EIGEN_ALWAYS_INLINE void colVecColLoopBodyExtraN(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
+                                                 const Packet4f pAlpha, float* result) {
+  if (MAX_BFLOAT16_VEC_ACC > num_acc) {
+    colVecColLoopBody<num_acc + (extraRows ? 1 : 0), LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs,
+                                                                                              pAlpha, result);
+  }
+}
+
+template <typename LhsMapper, typename RhsMapper, bool extraRows, bool linear>
+EIGEN_ALWAYS_INLINE void colVecColLoopBodyExtra(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
+                                                const Packet4f pAlpha, float* result) {
+  switch ((rows - row) >> 2) {
+    case 7:
+      colVecColLoopBodyExtraN<7, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 6:
+      colVecColLoopBodyExtraN<6, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 5:
+      colVecColLoopBodyExtraN<5, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 4:
+      colVecColLoopBodyExtraN<4, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 3:
+      colVecColLoopBodyExtraN<3, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 2:
+      colVecColLoopBodyExtraN<2, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 1:
+      colVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
+      break;
+    default:
+      if (extraRows) {
+        colVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
+      }
+      break;
+  }
+}
+
+template <typename LhsMapper, typename RhsMapper, bool linear>
+EIGEN_ALWAYS_INLINE void calcVecColLoops(Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha,
+                                         float* result) {
+  Index row = 0;
+  if (rows >= (MAX_BFLOAT16_VEC_ACC * 4)) {
+    colVecColLoopBody<MAX_BFLOAT16_VEC_ACC, LhsMapper, RhsMapper, false, linear>(row, cend, rows, lhs, rhs, pAlpha,
+                                                                                 result);
+    result += row;
+  }
+  if (rows & 3) {
+    colVecColLoopBodyExtra<LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
+  } else {
+    colVecColLoopBodyExtra<LhsMapper, RhsMapper, false, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
+  }
+}
+
+template <typename RhsMapper, typename LhsMapper, typename = void>
+struct UseMMAStride : std::false_type {
+  static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs, Packet4f pAlpha,
+                                      float* result) {
+    using RhsSubMapper = typename RhsMapper::SubMapper;
+
+    RhsSubMapper rhs2 = rhs.getSubMapper(j2, 0);
+    calcVecColLoops<LhsMapper, RhsSubMapper, false>(jend - j2, rows, lhs, rhs2, pAlpha, result);
+  }
+};
+
+template <typename RhsMapper, typename LhsMapper>
+struct UseMMAStride<RhsMapper, LhsMapper,
+                    std::enable_if_t<std::is_member_function_pointer<decltype(&RhsMapper::stride)>::value>>
+    : std::true_type {
+  static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs, Packet4f pAlpha,
+                                      float* result) {
+    using RhsSubMapper = typename RhsMapper::SubMapper;
+
+    RhsSubMapper rhs2 = rhs.getSubMapper(j2, 0);
+    if (rhs.stride() == 1) {
+      calcVecColLoops<LhsMapper, RhsSubMapper, true>(jend - j2, rows, lhs, rhs2, pAlpha, result);
+    } else {
+      calcVecColLoops<LhsMapper, RhsSubMapper, false>(jend - j2, rows, lhs, rhs2, pAlpha, result);
+    }
+  }
+};
+
+template <typename LhsMapper, typename RhsMapper>
+void gemvMMA_bfloat16_col(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, bfloat16* res,
+                          Index resIncr, bfloat16 alpha) {
+  EIGEN_UNUSED_VARIABLE(resIncr);
+  eigen_internal_assert(resIncr == 1);
+
+  // The following copy tells the compiler that lhs's attributes are not modified outside this function
+  // This helps GCC to generate proper code.
+  LhsMapper lhs(alhs);
+  RhsMapper rhs2(rhs);
+
+  const Index lhsStride = lhs.stride();
+
+  // TODO: improve the following heuristic:
+  const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(bfloat16) < 16000 ? 16 : 8);
+  float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
+  Packet4f pAlpha = pset1<Packet4f>(falpha);
+
+  ei_declare_aligned_stack_constructed_variable(float, result, rows, 0);
+
+  convertArrayPointerBF16toF32(result, 1, rows, res);
+
+  for (Index j2 = 0; j2 < cols; j2 += block_cols) {
+    Index jend = numext::mini(j2 + block_cols, cols);
+
+    using LhsSubMapper = typename LhsMapper::SubMapper;
+
+    LhsSubMapper lhs2 = lhs.getSubMapper(0, j2);
+    UseMMAStride<RhsMapper, LhsSubMapper>::run(j2, jend, rows, lhs2, rhs2, pAlpha, result);
+  }
+
+  convertArrayPointerF32toBF16(result, rows, res);
+}
+
+static Packet16uc p16uc_ELEMENT_VEC3 = {0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f,
+                                        0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f};
+
+template <Index num_acc>
+EIGEN_ALWAYS_INLINE void preduxVecResults2(Packet4f (&acc)[num_acc][4], Index k) {
+  if (num_acc > (k + 1)) {
+    acc[k][0] = vec_mergeh(acc[k][0], acc[k + 1][0]);
+    acc[k][1] = vec_mergeo(acc[k][1], acc[k + 1][1]);
+    acc[k][2] = vec_mergel(acc[k][2], acc[k + 1][2]);
+    acc[k][3] = vec_perm(acc[k][3], acc[k + 1][3], p16uc_ELEMENT_VEC3);
+
+    acc[k][0] = (acc[k][0] + acc[k][2]) + (acc[k][1] + acc[k][3]);
+  } else {
+    acc[k][0] = vec_mergeh(acc[k][0], acc[k][1]);
+    acc[k][0] += vec_mergel(acc[k][2], acc[k][3]);
+#ifdef _BIG_ENDIAN
+    acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12);
+#else
+    acc[k][0] += vec_sld(acc[k][0], acc[k][0], 4);
+#endif
+  }
+}
+
+template <Index num_acc>
+EIGEN_ALWAYS_INLINE void preduxVecResults(Packet4f (&acc)[num_acc][4]) {
+  BFLOAT16_UNROLL
+  for (Index k = 0; k < num_acc; k += 4) {
+    preduxVecResults2<num_acc>(acc, k + 0);
+    if (num_acc > (k + 2)) {
+      preduxVecResults2<num_acc>(acc, k + 2);
+      acc[k + 0][0] = reinterpret_cast<Packet4f>(
+          vec_mergeh(reinterpret_cast<Packet2ul>(acc[k + 0][0]), reinterpret_cast<Packet2ul>(acc[k + 2][0])));
+    }
+  }
+}
+
+template <Index num_acc, typename LhsMapper, typename RhsMapper, bool extra>
+EIGEN_ALWAYS_INLINE void multVecLoop(__vector_quad (&quad_acc)[num_acc], const LhsMapper& lhs, RhsMapper& rhs, Index j,
+                                     Index extra_cols) {
+  Packet8bf a0[num_acc], b0;
+
+  if (extra) {
+    b0 = rhs.template loadPacketPartial<Packet8bf>(j, extra_cols);
+  } else {
+    b0 = rhs.template loadPacket<Packet8bf>(j);
+  }
+
+  const LhsMapper lhs2 = lhs.getSubMapper(0, j);
+  BFLOAT16_UNROLL
+  for (Index k = 0; k < num_acc; k++) {
+    if (extra) {
+      a0[k] = lhs2.template loadPacketPartial<Packet8bf>(k, 0, extra_cols);
+    } else {
+      a0[k] = lhs2.template loadPacket<Packet8bf>(k, 0);
+    }
+  }
+
+  multVec<num_acc>(quad_acc, a0, b0);
+}
+
+template <Index num_acc, typename LhsMapper, typename RhsMapper>
+EIGEN_ALWAYS_INLINE void vecLoop(Index cols, const LhsMapper& lhs, RhsMapper& rhs, __vector_quad (&quad_acc)[num_acc],
+                                 Index extra_cols) {
+  Index j = 0;
+  for (; j + 8 <= cols; j += 8) {
+    multVecLoop<num_acc, LhsMapper, RhsMapper, false>(quad_acc, lhs, rhs, j, extra_cols);
+  }
+
+  if (extra_cols) {
+    multVecLoop<num_acc, LhsMapper, RhsMapper, true>(quad_acc, lhs, rhs, j, extra_cols);
+  }
+}
+
+template <const Index num_acc, typename LhsMapper, typename RhsMapper>
+void colVecLoopBody(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha,
+                    float* result) {
+  constexpr bool multiIters = (num_acc == MAX_BFLOAT16_VEC_ACC);
+  const Index extra_cols = (cols & 7);
+
+  do {
+    Packet4f acc[num_acc][4];
+    __vector_quad quad_acc[num_acc];
+
+    zeroAccumulators<num_acc>(quad_acc);
+
+    const LhsMapper lhs2 = lhs.getSubMapper(row, 0);
+    vecLoop<num_acc, LhsMapper, RhsMapper>(cols, lhs2, rhs, quad_acc, extra_cols);
+
+    disassembleAccumulators<num_acc>(quad_acc, acc);
+
+    preduxVecResults<num_acc>(acc);
+
+    outputVecResults<num_acc>(acc, result, pAlpha);
+
+    result += num_acc;
+  } while (multiIters && (num_acc <= rows - (row += num_acc)));
+}
+
+template <const Index num_acc, typename LhsMapper, typename RhsMapper>
+EIGEN_ALWAYS_INLINE void colVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs,
+                                              const Packet4f pAlpha, float* result) {
+  if (MAX_BFLOAT16_VEC_ACC > num_acc) {
+    colVecLoopBody<num_acc, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
+  }
+}
+
+template <typename LhsMapper, typename RhsMapper>
+EIGEN_ALWAYS_INLINE void colVecLoopBodyExtra(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs,
+                                             const Packet4f pAlpha, float* result) {
+  switch (rows - row) {
+    case 7:
+      colVecLoopBodyExtraN<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 6:
+      colVecLoopBodyExtraN<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 5:
+      colVecLoopBodyExtraN<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 4:
+      colVecLoopBodyExtraN<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 3:
+      colVecLoopBodyExtraN<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 2:
+      colVecLoopBodyExtraN<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
+      break;
+    case 1:
+      colVecLoopBodyExtraN<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
+      break;
+  }
+}
+
+template <typename LhsMapper, typename RhsMapper>
+EIGEN_ALWAYS_INLINE void calcVecLoops(Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha,
+                                      float* result) {
+  Index row = 0;
+  if (rows >= MAX_BFLOAT16_VEC_ACC) {
+    colVecLoopBody<MAX_BFLOAT16_VEC_ACC, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
+    result += row;
+  }
+  colVecLoopBodyExtra<LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
+}
+
+template <typename LhsMapper, typename RhsMapper>
+EIGEN_STRONG_INLINE void gemvMMA_bfloat16_row(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs,
+                                              bfloat16* res, Index resIncr, bfloat16 alpha) {
+  typedef typename RhsMapper::LinearMapper LinearMapper;
+
+  // The following copy tells the compiler that lhs's attributes are not modified outside this function
+  // This helps GCC to generate proper code.
+  LhsMapper lhs(alhs);
+  LinearMapper rhs2 = rhs.getLinearMapper(0, 0);
+
+  eigen_internal_assert(rhs.stride() == 1);
+
+  float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
+  const Packet4f pAlpha = pset1<Packet4f>(falpha);
+
+  ei_declare_aligned_stack_constructed_variable(float, result, rows, 0);
+  if (resIncr == 1) {
+    convertArrayPointerBF16toF32(result, 1, rows, res);
+  } else {
+    convertArrayPointerBF16toF32<true>(result, 1, rows, res, resIncr);
+  }
+  calcVecLoops<LhsMapper, LinearMapper>(cols, rows, lhs, rhs2, pAlpha, result);
+  if (resIncr == 1) {
+    convertArrayPointerF32toBF16(result, rows, res);
+  } else {
+    convertArrayPointerF32toBF16<true>(result, rows, res, resIncr);
+  }
+}
+#endif
+
+#undef MAX_BFLOAT16_VEC_ACC
+#undef BFLOAT16_UNROLL
+
+}  // namespace internal
+}  // namespace Eigen
+#endif  // EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H

include/Eigen/src/Core/arch/AltiVec/TypeCasting.h

@@ -0,0 +1,163 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2019 Rasmus Munk Larsen <rmlarsen@google.com>
+// Copyright (C) 2023 Chip Kerchner (chip.kerchner@ibm.com)
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_TYPE_CASTING_ALTIVEC_H
+#define EIGEN_TYPE_CASTING_ALTIVEC_H
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+
+namespace internal {
+template <>
+struct type_casting_traits<float, int> {
+  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
+};
+
+template <>
+struct type_casting_traits<int, float> {
+  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
+};
+
+template <>
+struct type_casting_traits<bfloat16, unsigned short int> {
+  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
+};
+
+template <>
+struct type_casting_traits<unsigned short int, bfloat16> {
+  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
+};
+
+template <>
+EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
+  return vec_cts(a, 0);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4ui pcast<Packet4f, Packet4ui>(const Packet4f& a) {
+  return vec_ctu(a, 0);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) {
+  return vec_ctf(a, 0);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4f pcast<Packet4ui, Packet4f>(const Packet4ui& a) {
+  return vec_ctf(a, 0);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8us pcast<Packet8bf, Packet8us>(const Packet8bf& a) {
+  Packet4f float_even = Bf16ToF32Even(a);
+  Packet4f float_odd = Bf16ToF32Odd(a);
+  Packet4ui int_even = pcast<Packet4f, Packet4ui>(float_even);
+  Packet4ui int_odd = pcast<Packet4f, Packet4ui>(float_odd);
+  const EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF);
+  Packet4ui low_even = pand<Packet4ui>(int_even, p4ui_low_mask);
+  Packet4ui low_odd = pand<Packet4ui>(int_odd, p4ui_low_mask);
+
+  // Check values that are bigger than USHRT_MAX (0xFFFF)
+  Packet4bi overflow_selector;
+  if (vec_any_gt(int_even, p4ui_low_mask)) {
+    overflow_selector = vec_cmpgt(int_even, p4ui_low_mask);
+    low_even = vec_sel(low_even, p4ui_low_mask, overflow_selector);
+  }
+  if (vec_any_gt(int_odd, p4ui_low_mask)) {
+    overflow_selector = vec_cmpgt(int_odd, p4ui_low_mask);
+    low_odd = vec_sel(low_even, p4ui_low_mask, overflow_selector);
+  }
+
+  return pmerge(low_even, low_odd);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet8bf pcast<Packet8us, Packet8bf>(const Packet8us& a) {
+  // short -> int -> float -> bfloat16
+  const EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF);
+  Packet4ui int_cast = reinterpret_cast<Packet4ui>(a);
+  Packet4ui int_even = pand<Packet4ui>(int_cast, p4ui_low_mask);
+  Packet4ui int_odd = plogical_shift_right<16>(int_cast);
+  Packet4f float_even = pcast<Packet4ui, Packet4f>(int_even);
+  Packet4f float_odd = pcast<Packet4ui, Packet4f>(int_odd);
+  return F32ToBf16(float_even, float_odd);
+}
+
+template <>
+struct type_casting_traits<bfloat16, float> {
+  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
+};
+
+template <>
+EIGEN_STRONG_INLINE Packet4f pcast<Packet8bf, Packet4f>(const Packet8bf& a) {
+  Packet8us z = pset1<Packet8us>(0);
+#ifdef _BIG_ENDIAN
+  return reinterpret_cast<Packet4f>(vec_mergeh(a.m_val, z));
+#else
+  return reinterpret_cast<Packet4f>(vec_mergeh(z, a.m_val));
+#endif
+}
+
+template <>
+struct type_casting_traits<float, bfloat16> {
+  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
+};
+
+template <>
+EIGEN_STRONG_INLINE Packet8bf pcast<Packet4f, Packet8bf>(const Packet4f& a, const Packet4f& b) {
+  return F32ToBf16Both(a, b);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4f>(const Packet4f& a) {
+  return reinterpret_cast<Packet4i>(a);
+}
+
+template <>
+EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4i>(const Packet4i& a) {
+  return reinterpret_cast<Packet4f>(a);
+}
+
+#ifdef EIGEN_VECTORIZE_VSX
+// VSX support varies between different compilers and even different
+// versions of the same compiler.  For gcc version >= 4.9.3, we can use
+// vec_cts to efficiently convert Packet2d to Packet2l.  Otherwise, use
+// a slow version that works with older compilers.
+// Update: apparently vec_cts/vec_ctf intrinsics for 64-bit doubles
+// are buggy, https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70963
+template <>
+inline Packet2l pcast<Packet2d, Packet2l>(const Packet2d& x) {
+#if EIGEN_GNUC_STRICT_AT_LEAST(7, 1, 0)
+  return vec_cts(x, 0);  // TODO: check clang version.
+#else
+  double tmp[2];
+  memcpy(tmp, &x, sizeof(tmp));
+  Packet2l l = {static_cast<long long>(tmp[0]), static_cast<long long>(tmp[1])};
+  return l;
+#endif
+}
+
+template <>
+inline Packet2d pcast<Packet2l, Packet2d>(const Packet2l& x) {
+  unsigned long long tmp[2];
+  memcpy(tmp, &x, sizeof(tmp));
+  Packet2d d = {static_cast<double>(tmp[0]), static_cast<double>(tmp[1])};
+  return d;
+}
+#endif
+
+}  // end namespace internal
+
+}  // end namespace Eigen
+
+#endif  // EIGEN_TYPE_CASTING_ALTIVEC_H

include/Eigen/src/Core/arch/Default/BFloat16.h

@@ -0,0 +1,855 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef EIGEN_BFLOAT16_H
+#define EIGEN_BFLOAT16_H
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+#if defined(EIGEN_HAS_HIP_BF16)
+// When compiling with GPU support, the "hip_bfloat16" base class as well as
+// some other routines are defined in the GPU compiler header files
+// (hip_bfloat16.h), and they are not tagged constexpr
+// As a consequence, we get compile failures when compiling Eigen with
+// GPU support. Hence the need to disable EIGEN_CONSTEXPR when building
+// Eigen with GPU support
+#pragma push_macro("EIGEN_CONSTEXPR")
+#undef EIGEN_CONSTEXPR
+#define EIGEN_CONSTEXPR
+#endif
+
+#define BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, METHOD)                                         \
+  template <>                                                                                       \
+  EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED PACKET_BF16 METHOD<PACKET_BF16>( \
+      const PACKET_BF16& _x) {                                                                      \
+    return F32ToBf16(METHOD<PACKET_F>(Bf16ToF32(_x)));                                              \
+  }
+
+// Only use HIP GPU bf16 in kernels
+#if defined(EIGEN_HAS_HIP_BF16) && defined(EIGEN_GPU_COMPILE_PHASE)
+#define EIGEN_USE_HIP_BF16
+#endif
+
+namespace Eigen {
+
+struct bfloat16;
+
+namespace numext {
+template <>
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 bit_cast<Eigen::bfloat16, uint16_t>(const uint16_t& src);
+
+template <>
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC uint16_t bit_cast<uint16_t, Eigen::bfloat16>(const Eigen::bfloat16& src);
+}  // namespace numext
+namespace bfloat16_impl {
+
+#if defined(EIGEN_USE_HIP_BF16)
+
+struct __bfloat16_raw : public hip_bfloat16 {
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() {}
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(hip_bfloat16 hb) : hip_bfloat16(hb) {}
+  explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(unsigned short raw) : hip_bfloat16(raw) {}
+};
+
+#else
+
+// Make our own __bfloat16_raw definition.
+struct __bfloat16_raw {
+#if defined(EIGEN_HAS_HIP_BF16) && !defined(EIGEN_GPU_COMPILE_PHASE)
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() {}
+#else
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() : value(0) {}
+#endif
+  explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(unsigned short raw) : value(raw) {}
+  unsigned short value;
+};
+
+#endif  // defined(EIGEN_USE_HIP_BF16)
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(unsigned short value);
+template <bool AssumeArgumentIsNormalOrInfinityOrZero>
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(float ff);
+// Forward declarations of template specializations, to avoid Visual C++ 2019 errors, saying:
+// > error C2908: explicit specialization; 'float_to_bfloat16_rtne' has already been instantiated
+template <>
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<false>(float ff);
+template <>
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<true>(float ff);
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float bfloat16_to_float(__bfloat16_raw h);
+
+struct bfloat16_base : public __bfloat16_raw {
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16_base() {}
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16_base(const __bfloat16_raw& h) : __bfloat16_raw(h) {}
+};
+
+}  // namespace bfloat16_impl
+
+// Class definition.
+struct bfloat16 : public bfloat16_impl::bfloat16_base {
+  typedef bfloat16_impl::__bfloat16_raw __bfloat16_raw;
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16() {}
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(const __bfloat16_raw& h) : bfloat16_impl::bfloat16_base(h) {}
+
+  explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(bool b)
+      : bfloat16_impl::bfloat16_base(bfloat16_impl::raw_uint16_to_bfloat16(b ? 0x3f80 : 0)) {}
+
+  template <class T>
+  explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(T val)
+      : bfloat16_impl::bfloat16_base(
+            bfloat16_impl::float_to_bfloat16_rtne<internal::is_integral<T>::value>(static_cast<float>(val))) {}
+
+  explicit EIGEN_DEVICE_FUNC bfloat16(float f)
+      : bfloat16_impl::bfloat16_base(bfloat16_impl::float_to_bfloat16_rtne<false>(f)) {}
+
+  // Following the convention of numpy, converting between complex and
+  // float will lead to loss of imag value.
+  template <typename RealScalar>
+  explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(const std::complex<RealScalar>& val)
+      : bfloat16_impl::bfloat16_base(bfloat16_impl::float_to_bfloat16_rtne<false>(static_cast<float>(val.real()))) {}
+
+  EIGEN_DEVICE_FUNC operator float() const {  // NOLINT: Allow implicit conversion to float, because it is lossless.
+    return bfloat16_impl::bfloat16_to_float(*this);
+  }
+};
+
+// TODO(majnemer): Get rid of this once we can rely on C++17 inline variables do
+// solve the ODR issue.
+namespace bfloat16_impl {
+template <typename = void>
+struct numeric_limits_bfloat16_impl {
+  static EIGEN_CONSTEXPR const bool is_specialized = true;
+  static EIGEN_CONSTEXPR const bool is_signed = true;
+  static EIGEN_CONSTEXPR const bool is_integer = false;
+  static EIGEN_CONSTEXPR const bool is_exact = false;
+  static EIGEN_CONSTEXPR const bool has_infinity = true;
+  static EIGEN_CONSTEXPR const bool has_quiet_NaN = true;
+  static EIGEN_CONSTEXPR const bool has_signaling_NaN = true;
+  EIGEN_DIAGNOSTICS(push)
+  EIGEN_DISABLE_DEPRECATED_WARNING
+  static EIGEN_CONSTEXPR const std::float_denorm_style has_denorm = std::denorm_present;
+  static EIGEN_CONSTEXPR const bool has_denorm_loss = false;
+  EIGEN_DIAGNOSTICS(pop)
+  static EIGEN_CONSTEXPR const std::float_round_style round_style = std::numeric_limits<float>::round_style;
+  static EIGEN_CONSTEXPR const bool is_iec559 = true;
+  // The C++ standard defines this as "true if the set of values representable
+  // by the type is finite." BFloat16 has finite precision.
+  static EIGEN_CONSTEXPR const bool is_bounded = true;
+  static EIGEN_CONSTEXPR const bool is_modulo = false;
+  static EIGEN_CONSTEXPR const int digits = 8;
+  static EIGEN_CONSTEXPR const int digits10 = 2;
+  static EIGEN_CONSTEXPR const int max_digits10 = 4;
+  static EIGEN_CONSTEXPR const int radix = std::numeric_limits<float>::radix;
+  static EIGEN_CONSTEXPR const int min_exponent = std::numeric_limits<float>::min_exponent;
+  static EIGEN_CONSTEXPR const int min_exponent10 = std::numeric_limits<float>::min_exponent10;
+  static EIGEN_CONSTEXPR const int max_exponent = std::numeric_limits<float>::max_exponent;
+  static EIGEN_CONSTEXPR const int max_exponent10 = std::numeric_limits<float>::max_exponent10;
+  static EIGEN_CONSTEXPR const bool traps = std::numeric_limits<float>::traps;
+  // IEEE754: "The implementer shall choose how tininess is detected, but shall
+  // detect tininess in the same way for all operations in radix two"
+  static EIGEN_CONSTEXPR const bool tinyness_before = std::numeric_limits<float>::tinyness_before;
+
+  static EIGEN_CONSTEXPR Eigen::bfloat16(min)() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0080); }
+  static EIGEN_CONSTEXPR Eigen::bfloat16 lowest() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0xff7f); }
+  static EIGEN_CONSTEXPR Eigen::bfloat16(max)() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f7f); }
+  static EIGEN_CONSTEXPR Eigen::bfloat16 epsilon() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x3c00); }
+  static EIGEN_CONSTEXPR Eigen::bfloat16 round_error() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x3f00); }
+  static EIGEN_CONSTEXPR Eigen::bfloat16 infinity() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f80); }
+  static EIGEN_CONSTEXPR Eigen::bfloat16 quiet_NaN() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7fc0); }
+  static EIGEN_CONSTEXPR Eigen::bfloat16 signaling_NaN() {
+    return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7fa0);
+  }
+  static EIGEN_CONSTEXPR Eigen::bfloat16 denorm_min() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0001); }
+};
+
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_specialized;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_signed;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_integer;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_exact;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::has_infinity;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::has_quiet_NaN;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::has_signaling_NaN;
+EIGEN_DIAGNOSTICS(push)
+EIGEN_DISABLE_DEPRECATED_WARNING
+template <typename T>
+EIGEN_CONSTEXPR const std::float_denorm_style numeric_limits_bfloat16_impl<T>::has_denorm;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::has_denorm_loss;
+EIGEN_DIAGNOSTICS(pop)
+template <typename T>
+EIGEN_CONSTEXPR const std::float_round_style numeric_limits_bfloat16_impl<T>::round_style;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_iec559;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_bounded;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_modulo;
+template <typename T>
+EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::digits;
+template <typename T>
+EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::digits10;
+template <typename T>
+EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::max_digits10;
+template <typename T>
+EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::radix;
+template <typename T>
+EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::min_exponent;
+template <typename T>
+EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::min_exponent10;
+template <typename T>
+EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::max_exponent;
+template <typename T>
+EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::max_exponent10;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::traps;
+template <typename T>
+EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::tinyness_before;
+}  // end namespace bfloat16_impl
+}  // end namespace Eigen
+
+namespace std {
+// If std::numeric_limits<T> is specialized, should also specialize
+// std::numeric_limits<const T>, std::numeric_limits<volatile T>, and
+// std::numeric_limits<const volatile T>
+// https://stackoverflow.com/a/16519653/
+template <>
+class numeric_limits<Eigen::bfloat16> : public Eigen::bfloat16_impl::numeric_limits_bfloat16_impl<> {};
+template <>
+class numeric_limits<const Eigen::bfloat16> : public numeric_limits<Eigen::bfloat16> {};
+template <>
+class numeric_limits<volatile Eigen::bfloat16> : public numeric_limits<Eigen::bfloat16> {};
+template <>
+class numeric_limits<const volatile Eigen::bfloat16> : public numeric_limits<Eigen::bfloat16> {};
+}  // end namespace std
+
+namespace Eigen {
+
+namespace bfloat16_impl {
+
+// We need to distinguish ‘clang as the CUDA compiler’ from ‘clang as the host compiler,
+// invoked by NVCC’ (e.g. on MacOS). The former needs to see both host and device implementation
+// of the functions, while the latter can only deal with one of them.
+#if !defined(EIGEN_HAS_NATIVE_BF16) || (EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC)  // Emulate support for bfloat16 floats
+
+#if EIGEN_COMP_CLANG && defined(EIGEN_CUDACC)
+// We need to provide emulated *host-side* BF16 operators for clang.
+#pragma push_macro("EIGEN_DEVICE_FUNC")
+#undef EIGEN_DEVICE_FUNC
+#if (defined(EIGEN_HAS_GPU_BF16) && defined(EIGEN_HAS_NATIVE_BF16))
+#define EIGEN_DEVICE_FUNC __host__
+#else  // both host and device need emulated ops.
+#define EIGEN_DEVICE_FUNC __host__ __device__
+#endif
+#endif
+
+// Definitions for CPUs, mostly working through conversion
+// to/from fp32.
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator+(const bfloat16& a, const bfloat16& b) {
+  return bfloat16(float(a) + float(b));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator+(const bfloat16& a, const int& b) {
+  return bfloat16(float(a) + static_cast<float>(b));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator+(const int& a, const bfloat16& b) {
+  return bfloat16(static_cast<float>(a) + float(b));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator*(const bfloat16& a, const bfloat16& b) {
+  return bfloat16(float(a) * float(b));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator-(const bfloat16& a, const bfloat16& b) {
+  return bfloat16(float(a) - float(b));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator/(const bfloat16& a, const bfloat16& b) {
+  return bfloat16(float(a) / float(b));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator-(const bfloat16& a) {
+  numext::uint16_t x = numext::bit_cast<uint16_t>(a) ^ 0x8000;
+  return numext::bit_cast<bfloat16>(x);
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator+=(bfloat16& a, const bfloat16& b) {
+  a = bfloat16(float(a) + float(b));
+  return a;
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator*=(bfloat16& a, const bfloat16& b) {
+  a = bfloat16(float(a) * float(b));
+  return a;
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator-=(bfloat16& a, const bfloat16& b) {
+  a = bfloat16(float(a) - float(b));
+  return a;
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator/=(bfloat16& a, const bfloat16& b) {
+  a = bfloat16(float(a) / float(b));
+  return a;
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator++(bfloat16& a) {
+  a += bfloat16(1);
+  return a;
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator--(bfloat16& a) {
+  a -= bfloat16(1);
+  return a;
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator++(bfloat16& a, int) {
+  bfloat16 original_value = a;
+  ++a;
+  return original_value;
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator--(bfloat16& a, int) {
+  bfloat16 original_value = a;
+  --a;
+  return original_value;
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator==(const bfloat16& a, const bfloat16& b) {
+  return numext::equal_strict(float(a), float(b));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator!=(const bfloat16& a, const bfloat16& b) {
+  return numext::not_equal_strict(float(a), float(b));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<(const bfloat16& a, const bfloat16& b) {
+  return float(a) < float(b);
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<=(const bfloat16& a, const bfloat16& b) {
+  return float(a) <= float(b);
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>(const bfloat16& a, const bfloat16& b) {
+  return float(a) > float(b);
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>=(const bfloat16& a, const bfloat16& b) {
+  return float(a) >= float(b);
+}
+
+#if EIGEN_COMP_CLANG && defined(EIGEN_CUDACC)
+#pragma pop_macro("EIGEN_DEVICE_FUNC")
+#endif
+#endif  // Emulate support for bfloat16 floats
+
+// Division by an index. Do it in full float precision to avoid accuracy
+// issues in converting the denominator to bfloat16.
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator/(const bfloat16& a, Index b) {
+  return bfloat16(static_cast<float>(a) / static_cast<float>(b));
+}
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw truncate_to_bfloat16(const float v) {
+#if defined(EIGEN_USE_HIP_BF16)
+  return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(v, __bfloat16_raw::truncate));
+#else
+  __bfloat16_raw output;
+  if (numext::isnan EIGEN_NOT_A_MACRO(v)) {
+    output.value = std::signbit(v) ? 0xFFC0 : 0x7FC0;
+    return output;
+  }
+  output.value = static_cast<numext::uint16_t>(numext::bit_cast<numext::uint32_t>(v) >> 16);
+  return output;
+#endif
+}
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(numext::uint16_t value) {
+#if defined(EIGEN_USE_HIP_BF16)
+  __bfloat16_raw bf;
+  bf.data = value;
+  return bf;
+#else
+  return __bfloat16_raw(value);
+#endif
+}
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR numext::uint16_t raw_bfloat16_as_uint16(
+    const __bfloat16_raw& bf) {
+#if defined(EIGEN_USE_HIP_BF16)
+  return bf.data;
+#else
+  return bf.value;
+#endif
+}
+
+// float_to_bfloat16_rtne template specialization that does not make any
+// assumption about the value of its function argument (ff).
+template <>
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<false>(float ff) {
+#if defined(EIGEN_USE_HIP_BF16)
+  return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(ff));
+#else
+  __bfloat16_raw output;
+
+  if (numext::isnan EIGEN_NOT_A_MACRO(ff)) {
+    // If the value is a NaN, squash it to a qNaN with msb of fraction set,
+    // this makes sure after truncation we don't end up with an inf.
+    //
+    // qNaN magic: All exponent bits set + most significant bit of fraction
+    // set.
+    output.value = std::signbit(ff) ? 0xFFC0 : 0x7FC0;
+  } else {
+    // Fast rounding algorithm that rounds a half value to nearest even. This
+    // reduces expected error when we convert a large number of floats. Here
+    // is how it works:
+    //
+    // Definitions:
+    // To convert a float 32 to bfloat16, a float 32 can be viewed as 32 bits
+    // with the following tags:
+    //
+    // Sign |  Exp (8 bits) | Frac (23 bits)
+    //  S     EEEEEEEE         FFFFFFLRTTTTTTTTTTTTTTT
+    //
+    //  S: Sign bit.
+    //  E: Exponent bits.
+    //  F: First 6 bits of fraction.
+    //  L: Least significant bit of resulting bfloat16 if we truncate away the
+    //  rest of the float32. This is also the 7th bit of fraction
+    //  R: Rounding bit, 8th bit of fraction.
+    //  T: Sticky bits, rest of fraction, 15 bits.
+    //
+    // To round half to nearest even, there are 3 cases where we want to round
+    // down (simply truncate the result of the bits away, which consists of
+    // rounding bit and sticky bits) and two cases where we want to round up
+    // (truncate then add one to the result).
+    //
+    // The fast converting algorithm simply adds lsb (L) to 0x7fff (15 bits of
+    // 1s) as the rounding bias, adds the rounding bias to the input, then
+    // truncates the last 16 bits away.
+    //
+    // To understand how it works, we can analyze this algorithm case by case:
+    //
+    // 1. L = 0, R = 0:
+    //   Expect: round down, this is less than half value.
+    //
+    //   Algorithm:
+    //   - Rounding bias: 0x7fff + 0 = 0x7fff
+    //   - Adding rounding bias to input may create any carry, depending on
+    //   whether there is any value set to 1 in T bits.
+    //   - R may be set to 1 if there is a carry.
+    //   - L remains 0.
+    //   - Note that this case also handles Inf and -Inf, where all fraction
+    //   bits, including L, R and Ts are all 0. The output remains Inf after
+    //   this algorithm.
+    //
+    // 2. L = 1, R = 0:
+    //   Expect: round down, this is less than half value.
+    //
+    //   Algorithm:
+    //   - Rounding bias: 0x7fff + 1 = 0x8000
+    //   - Adding rounding bias to input doesn't change sticky bits but
+    //   adds 1 to rounding bit.
+    //   - L remains 1.
+    //
+    // 3. L = 0, R = 1, all of T are 0:
+    //   Expect: round down, this is exactly at half, the result is already
+    //   even (L=0).
+    //
+    //   Algorithm:
+    //   - Rounding bias: 0x7fff + 0 = 0x7fff
+    //   - Adding rounding bias to input sets all sticky bits to 1, but
+    //   doesn't create a carry.
+    //   - R remains 1.
+    //   - L remains 0.
+    //
+    // 4. L = 1, R = 1:
+    //   Expect: round up, this is exactly at half, the result needs to be
+    //   round to the next even number.
+    //
+    //   Algorithm:
+    //   - Rounding bias: 0x7fff + 1 = 0x8000
+    //   - Adding rounding bias to input doesn't change sticky bits, but
+    //   creates a carry from rounding bit.
+    //   - The carry sets L to 0, creates another carry bit and propagate
+    //   forward to F bits.
+    //   - If all the F bits are 1, a carry then propagates to the exponent
+    //   bits, which then creates the minimum value with the next exponent
+    //   value. Note that we won't have the case where exponents are all 1,
+    //   since that's either a NaN (handled in the other if condition) or inf
+    //   (handled in case 1).
+    //
+    // 5. L = 0, R = 1, any of T is 1:
+    //   Expect: round up, this is greater than half.
+    //
+    //   Algorithm:
+    //   - Rounding bias: 0x7fff + 0 = 0x7fff
+    //   - Adding rounding bias to input creates a carry from sticky bits,
+    //   sets rounding bit to 0, then create another carry.
+    //   - The second carry sets L to 1.
+    //
+    // Examples:
+    //
+    //  Exact half value that is already even:
+    //    Input:
+    //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+    //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+    //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 1 0     1000000000000000
+    //
+    //     This falls into case 3. We truncate the rest of 16 bits and no
+    //     carry is created into F and L:
+    //
+    //    Output:
+    //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+    //     S     E E E E E E E E      F F F F F F L
+    //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 1 0
+    //
+    //  Exact half value, round to next even number:
+    //    Input:
+    //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+    //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+    //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 0 1     1000000000000000
+    //
+    //     This falls into case 4. We create a carry from R and T,
+    //     which then propagates into L and F:
+    //
+    //    Output:
+    //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+    //     S     E E E E E E E E      F F F F F F L
+    //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 1 0
+    //
+    //
+    //  Max denormal value round to min normal value:
+    //    Input:
+    //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+    //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+    //     0     0 0 0 0 0 0 0 0      1 1 1 1 1 1 1     1111111111111111
+    //
+    //     This falls into case 4. We create a carry from R and T,
+    //     propagate into L and F, which then propagates into exponent
+    //     bits:
+    //
+    //    Output:
+    //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+    //     S     E E E E E E E E      F F F F F F L
+    //     0     0 0 0 0 0 0 0 1      0 0 0 0 0 0 0
+    //
+    //  Max normal value round to Inf:
+    //    Input:
+    //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+    //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+    //     0     1 1 1 1 1 1 1 0      1 1 1 1 1 1 1     1111111111111111
+    //
+    //     This falls into case 4. We create a carry from R and T,
+    //     propagate into L and F, which then propagates into exponent
+    //     bits:
+    //
+    //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+    //     S     E E E E E E E E      F F F F F F L
+    //     0     1 1 1 1 1 1 1 1      0 0 0 0 0 0 0
+
+    // At this point, ff must be either a normal float, or +/-infinity.
+    output = float_to_bfloat16_rtne<true>(ff);
+  }
+  return output;
+#endif
+}
+
+// float_to_bfloat16_rtne template specialization that assumes that its function
+// argument (ff) is either a normal floating point number, or +/-infinity, or
+// zero. Used to improve the runtime performance of conversion from an integer
+// type to bfloat16.
+template <>
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<true>(float ff) {
+#if defined(EIGEN_USE_HIP_BF16)
+  return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(ff));
+#else
+  numext::uint32_t input = numext::bit_cast<numext::uint32_t>(ff);
+  __bfloat16_raw output;
+
+  // Least significant bit of resulting bfloat.
+  numext::uint32_t lsb = (input >> 16) & 1;
+  numext::uint32_t rounding_bias = 0x7fff + lsb;
+  input += rounding_bias;
+  output.value = static_cast<numext::uint16_t>(input >> 16);
+  return output;
+#endif
+}
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float bfloat16_to_float(__bfloat16_raw h) {
+#if defined(EIGEN_USE_HIP_BF16)
+  return static_cast<float>(h);
+#else
+  return numext::bit_cast<float>(static_cast<numext::uint32_t>(h.value) << 16);
+#endif
+}
+
+// --- standard functions ---
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isinf)(const bfloat16& a) {
+  EIGEN_USING_STD(isinf);
+#if defined(EIGEN_USE_HIP_BF16)
+  return (isinf)(a);  // Uses HIP hip_bfloat16 isinf operator
+#else
+  return (isinf)(float(a));
+#endif
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isnan)(const bfloat16& a) {
+  EIGEN_USING_STD(isnan);
+#if defined(EIGEN_USE_HIP_BF16)
+  return (isnan)(a);  // Uses HIP hip_bfloat16 isnan operator
+#else
+  return (isnan)(float(a));
+#endif
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isfinite)(const bfloat16& a) {
+  return !(isinf EIGEN_NOT_A_MACRO(a)) && !(isnan EIGEN_NOT_A_MACRO(a));
+}
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 abs(const bfloat16& a) {
+  numext::uint16_t x = numext::bit_cast<numext::uint16_t>(a) & 0x7FFF;
+  return numext::bit_cast<bfloat16>(x);
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 exp(const bfloat16& a) { return bfloat16(::expf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 exp2(const bfloat16& a) { return bfloat16(::exp2f(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 expm1(const bfloat16& a) { return bfloat16(numext::expm1(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log(const bfloat16& a) { return bfloat16(::logf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log1p(const bfloat16& a) { return bfloat16(numext::log1p(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log10(const bfloat16& a) { return bfloat16(::log10f(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log2(const bfloat16& a) {
+  return bfloat16(static_cast<float>(EIGEN_LOG2E) * ::logf(float(a)));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sqrt(const bfloat16& a) { return bfloat16(::sqrtf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 pow(const bfloat16& a, const bfloat16& b) {
+  return bfloat16(::powf(float(a), float(b)));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atan2(const bfloat16& a, const bfloat16& b) {
+  return bfloat16(::atan2f(float(a), float(b)));
+}
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sin(const bfloat16& a) { return bfloat16(::sinf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cos(const bfloat16& a) { return bfloat16(::cosf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 tan(const bfloat16& a) { return bfloat16(::tanf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 asin(const bfloat16& a) { return bfloat16(::asinf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 acos(const bfloat16& a) { return bfloat16(::acosf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atan(const bfloat16& a) { return bfloat16(::atanf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sinh(const bfloat16& a) { return bfloat16(::sinhf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cosh(const bfloat16& a) { return bfloat16(::coshf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 tanh(const bfloat16& a) { return bfloat16(::tanhf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 asinh(const bfloat16& a) { return bfloat16(::asinhf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 acosh(const bfloat16& a) { return bfloat16(::acoshf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atanh(const bfloat16& a) { return bfloat16(::atanhf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 floor(const bfloat16& a) { return bfloat16(::floorf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 ceil(const bfloat16& a) { return bfloat16(::ceilf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 rint(const bfloat16& a) { return bfloat16(::rintf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 round(const bfloat16& a) { return bfloat16(::roundf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 trunc(const bfloat16& a) { return bfloat16(::truncf(float(a))); }
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmod(const bfloat16& a, const bfloat16& b) {
+  return bfloat16(::fmodf(float(a), float(b)));
+}
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16(min)(const bfloat16& a, const bfloat16& b) {
+  const float f1 = static_cast<float>(a);
+  const float f2 = static_cast<float>(b);
+  return f2 < f1 ? b : a;
+}
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16(max)(const bfloat16& a, const bfloat16& b) {
+  const float f1 = static_cast<float>(a);
+  const float f2 = static_cast<float>(b);
+  return f1 < f2 ? b : a;
+}
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmin(const bfloat16& a, const bfloat16& b) {
+  const float f1 = static_cast<float>(a);
+  const float f2 = static_cast<float>(b);
+  return bfloat16(::fminf(f1, f2));
+}
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmax(const bfloat16& a, const bfloat16& b) {
+  const float f1 = static_cast<float>(a);
+  const float f2 = static_cast<float>(b);
+  return bfloat16(::fmaxf(f1, f2));
+}
+
+#ifndef EIGEN_NO_IO
+EIGEN_ALWAYS_INLINE std::ostream& operator<<(std::ostream& os, const bfloat16& v) {
+  os << static_cast<float>(v);
+  return os;
+}
+#endif
+
+}  // namespace bfloat16_impl
+
+namespace internal {
+
+template <>
+struct is_arithmetic<bfloat16> {
+  enum { value = true };
+};
+
+template <>
+struct random_impl<bfloat16> {
+  enum : int { MantissaBits = 7 };
+  using Impl = random_impl<float>;
+  static EIGEN_DEVICE_FUNC inline bfloat16 run(const bfloat16& x, const bfloat16& y) {
+    float result = Impl::run(x, y, MantissaBits);
+    return bfloat16(result);
+  }
+  static EIGEN_DEVICE_FUNC inline bfloat16 run() {
+    float result = Impl::run(MantissaBits);
+    return bfloat16(result);
+  }
+};
+
+}  // namespace internal
+
+template <>
+struct NumTraits<Eigen::bfloat16> : GenericNumTraits<Eigen::bfloat16> {
+  enum { IsSigned = true, IsInteger = false, IsComplex = false, RequireInitialization = false };
+
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 epsilon() {
+    return bfloat16_impl::raw_uint16_to_bfloat16(0x3c00);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 dummy_precision() {
+    return bfloat16_impl::raw_uint16_to_bfloat16(0x3D4D);  // bfloat16(5e-2f);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 highest() {
+    return bfloat16_impl::raw_uint16_to_bfloat16(0x7F7F);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 lowest() {
+    return bfloat16_impl::raw_uint16_to_bfloat16(0xFF7F);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 infinity() {
+    return bfloat16_impl::raw_uint16_to_bfloat16(0x7f80);
+  }
+  EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 quiet_NaN() {
+    return bfloat16_impl::raw_uint16_to_bfloat16(0x7fc0);
+  }
+};
+
+}  // namespace Eigen
+
+#if defined(EIGEN_HAS_HIP_BF16)
+#pragma pop_macro("EIGEN_CONSTEXPR")
+#endif
+
+namespace Eigen {
+namespace numext {
+
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isnan)(const Eigen::bfloat16& h) {
+  return (bfloat16_impl::isnan)(h);
+}
+
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isinf)(const Eigen::bfloat16& h) {
+  return (bfloat16_impl::isinf)(h);
+}
+
+template <>
+EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isfinite)(const Eigen::bfloat16& h) {
+  return (bfloat16_impl::isfinite)(h);
+}
+
+template <>
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 bit_cast<Eigen::bfloat16, uint16_t>(const uint16_t& src) {
+  return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(src);
+}
+
+template <>
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC uint16_t bit_cast<uint16_t, Eigen::bfloat16>(const Eigen::bfloat16& src) {
+  return Eigen::bfloat16_impl::raw_bfloat16_as_uint16(src);
+}
+
+EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 nextafter(const bfloat16& from, const bfloat16& to) {
+  if (numext::isnan EIGEN_NOT_A_MACRO(from)) {
+    return from;
+  }
+  if (numext::isnan EIGEN_NOT_A_MACRO(to)) {
+    return to;
+  }
+  if (from == to) {
+    return to;
+  }
+  uint16_t from_bits = numext::bit_cast<uint16_t>(from);
+  bool from_sign = from_bits >> 15;
+  // Whether we are adjusting toward the infinity with the same sign as from.
+  bool toward_inf = (to > from) == !from_sign;
+  if (toward_inf) {
+    ++from_bits;
+  } else if ((from_bits & 0x7fff) == 0) {
+    // Adjusting away from inf, but from is zero, so just toggle the sign.
+    from_bits ^= 0x8000;
+  } else {
+    --from_bits;
+  }
+  return numext::bit_cast<bfloat16>(from_bits);
+}
+
+}  // namespace numext
+}  // namespace Eigen
+
+#if EIGEN_HAS_STD_HASH
+namespace std {
+template <>
+struct hash<Eigen::bfloat16> {
+  EIGEN_STRONG_INLINE std::size_t operator()(const Eigen::bfloat16& a) const {
+    return static_cast<std::size_t>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(a));
+  }
+};
+}  // namespace std
+#endif
+
+// Add the missing shfl* intrinsics.
+// The __shfl* functions are only valid on HIP or _CUDA_ARCH_ >= 300.
+//   CUDA defines them for (__CUDA_ARCH__ >= 300 || !defined(__CUDA_ARCH__))
+//
+// HIP and CUDA prior to SDK 9.0 define
+//    __shfl, __shfl_up, __shfl_down, __shfl_xor for int and float
+// CUDA since 9.0 deprecates those and instead defines
+//    __shfl_sync, __shfl_up_sync, __shfl_down_sync, __shfl_xor_sync,
+//    with native support for __half and __nv_bfloat16
+//
+// Note that the following are __device__ - only functions.
+#if defined(EIGEN_HIPCC)
+
+#if defined(EIGEN_HAS_HIP_BF16)
+
+__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl(Eigen::bfloat16 var, int srcLane, int width = warpSize) {
+  const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
+  return Eigen::numext::bit_cast<Eigen::bfloat16>(static_cast<Eigen::numext::uint16_t>(__shfl(ivar, srcLane, width)));
+}
+
+__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_up(Eigen::bfloat16 var, unsigned int delta,
+                                                         int width = warpSize) {
+  const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
+  return Eigen::numext::bit_cast<Eigen::bfloat16>(static_cast<Eigen::numext::uint16_t>(__shfl_up(ivar, delta, width)));
+}
+
+__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_down(Eigen::bfloat16 var, unsigned int delta,
+                                                           int width = warpSize) {
+  const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
+  return Eigen::numext::bit_cast<Eigen::bfloat16>(
+      static_cast<Eigen::numext::uint16_t>(__shfl_down(ivar, delta, width)));
+}
+
+__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_xor(Eigen::bfloat16 var, int laneMask, int width = warpSize) {
+  const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
+  return Eigen::numext::bit_cast<Eigen::bfloat16>(
+      static_cast<Eigen::numext::uint16_t>(__shfl_xor(ivar, laneMask, width)));
+}
+
+#endif  // HIP
+
+#endif  // __shfl*
+
+#if defined(EIGEN_HIPCC)
+EIGEN_STRONG_INLINE __device__ Eigen::bfloat16 __ldg(const Eigen::bfloat16* ptr) {
+  return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(
+      __ldg(Eigen::numext::bit_cast<const Eigen::numext::uint16_t*>(ptr)));
+}
+#endif  // __ldg
+
+#endif  // EIGEN_BFLOAT16_H

include/Eigen/src/Core/arch/Default/ConjHelper.h

@@ -0,0 +1,128 @@
+
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2017 Gael Guennebaud <gael.guennebaud@inria.fr>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef EIGEN_ARCH_CONJ_HELPER_H
+#define EIGEN_ARCH_CONJ_HELPER_H
+
+#define EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(PACKET_CPLX, PACKET_REAL)                                                  \
+  template <>                                                                                                       \
+  struct conj_helper<PACKET_REAL, PACKET_CPLX, false, false> {                                                      \
+    EIGEN_STRONG_INLINE PACKET_CPLX pmadd(const PACKET_REAL& x, const PACKET_CPLX& y, const PACKET_CPLX& c) const { \
+      return padd(c, this->pmul(x, y));                                                                             \
+    }                                                                                                               \
+    EIGEN_STRONG_INLINE PACKET_CPLX pmul(const PACKET_REAL& x, const PACKET_CPLX& y) const {                        \
+      return PACKET_CPLX(Eigen::internal::pmul<PACKET_REAL>(x, y.v));                                               \
+    }                                                                                                               \
+  };                                                                                                                \
+                                                                                                                    \
+  template <>                                                                                                       \
+  struct conj_helper<PACKET_CPLX, PACKET_REAL, false, false> {                                                      \
+    EIGEN_STRONG_INLINE PACKET_CPLX pmadd(const PACKET_CPLX& x, const PACKET_REAL& y, const PACKET_CPLX& c) const { \
+      return padd(c, this->pmul(x, y));                                                                             \
+    }                                                                                                               \
+    EIGEN_STRONG_INLINE PACKET_CPLX pmul(const PACKET_CPLX& x, const PACKET_REAL& y) const {                        \
+      return PACKET_CPLX(Eigen::internal::pmul<PACKET_REAL>(x.v, y));                                               \
+    }                                                                                                               \
+  };
+
+// IWYU pragma: private
+#include "../../InternalHeaderCheck.h"
+
+namespace Eigen {
+namespace internal {
+
+template <bool Conjugate>
+struct conj_if;
+
+template <>
+struct conj_if<true> {
+  template <typename T>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& x) const {
+    return numext::conj(x);
+  }
+  template <typename T>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T pconj(const T& x) const {
+    return internal::pconj(x);
+  }
+};
+
+template <>
+struct conj_if<false> {
+  template <typename T>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& operator()(const T& x) const {
+    return x;
+  }
+  template <typename T>
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& pconj(const T& x) const {
+    return x;
+  }
+};
+
+// Generic Implementation, assume scalars since the packet-version is
+// specialized below.
+template <typename LhsType, typename RhsType, bool ConjLhs, bool ConjRhs>
+struct conj_helper {
+  typedef typename ScalarBinaryOpTraits<LhsType, RhsType>::ReturnType ResultType;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ResultType pmadd(const LhsType& x, const RhsType& y,
+                                                         const ResultType& c) const {
+    return this->pmul(x, y) + c;
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ResultType pmul(const LhsType& x, const RhsType& y) const {
+    return conj_if<ConjLhs>()(x) * conj_if<ConjRhs>()(y);
+  }
+};
+
+template <typename LhsScalar, typename RhsScalar>
+struct conj_helper<LhsScalar, RhsScalar, true, true> {
+  typedef typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType ResultType;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ResultType pmadd(const LhsScalar& x, const RhsScalar& y,
+                                                         const ResultType& c) const {
+    return this->pmul(x, y) + c;
+  }
+
+  // We save a conjuation by using the identity conj(a)*conj(b) = conj(a*b).
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ResultType pmul(const LhsScalar& x, const RhsScalar& y) const {
+    return numext::conj(x * y);
+  }
+};
+
+// Implementation with equal type, use packet operations.
+template <typename Packet, bool ConjLhs, bool ConjRhs>
+struct conj_helper<Packet, Packet, ConjLhs, ConjRhs> {
+  typedef Packet ResultType;
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pmadd(const Packet& x, const Packet& y, const Packet& c) const {
+    return Eigen::internal::pmadd(conj_if<ConjLhs>().pconj(x), conj_if<ConjRhs>().pconj(y), c);
+  }
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pmul(const Packet& x, const Packet& y) const {
+    return Eigen::internal::pmul(conj_if<ConjLhs>().pconj(x), conj_if<ConjRhs>().pconj(y));
+  }
+};
+
+template <typename Packet>
+struct conj_helper<Packet, Packet, true, true> {
+  typedef Packet ResultType;
+
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pmadd(const Packet& x, const Packet& y, const Packet& c) const {
+    return Eigen::internal::pmadd(pconj(x), pconj(y), c);
+  }
+  // We save a conjuation by using the identity conj(a)*conj(b) = conj(a*b).
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pmul(const Packet& x, const Packet& y) const {
+    return pconj(Eigen::internal::pmul(x, y));
+  }
+};
+
+}  // namespace internal
+}  // namespace Eigen
+
+#endif  // EIGEN_ARCH_CONJ_HELPER_H