Ginkgo  Generated from pipelines/2171896597 branch based on develop. Ginkgo version 1.11.0
A numerical linear algebra library targeting many-core architectures
dense.hpp
1 // SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors
2 //
3 // SPDX-License-Identifier: BSD-3-Clause
4 
5 #ifndef GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
6 #define GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
7 
8 
9 #include <initializer_list>
10 #include <type_traits>
11 
12 #include <ginkgo/core/base/array.hpp>
13 #include <ginkgo/core/base/exception_helpers.hpp>
14 #include <ginkgo/core/base/executor.hpp>
15 #include <ginkgo/core/base/lin_op.hpp>
16 #include <ginkgo/core/base/range_accessors.hpp>
17 #include <ginkgo/core/base/types.hpp>
18 #include <ginkgo/core/base/utils.hpp>
19 #include <ginkgo/core/matrix/permutation.hpp>
20 #include <ginkgo/core/matrix/scaled_permutation.hpp>
21 
22 
23 namespace gko {
24 namespace experimental {
25 namespace distributed {
26 
27 
28 template <typename ValueType>
29 class Vector;
30 
31 
32 namespace detail {
33 
34 
35 template <typename ValueType>
36 class VectorCache;
37 
38 
39 } // namespace detail
40 } // namespace distributed
41 } // namespace experimental
42 
43 
44 namespace matrix {
45 
46 
47 template <typename ValueType, typename IndexType>
48 class Coo;
49 
50 template <typename ValueType, typename IndexType>
51 class Csr;
52 
53 template <typename ValueType>
54 class Diagonal;
55 
56 template <typename ValueType, typename IndexType>
57 class Ell;
58 
59 template <typename ValueType, typename IndexType>
60 class Fbcsr;
61 
62 template <typename ValueType, typename IndexType>
63 class Hybrid;
64 
65 template <typename ValueType, typename IndexType>
66 class Sellp;
67 
68 template <typename ValueType, typename IndexType>
69 class SparsityCsr;
70 
71 
87 template <typename ValueType = default_precision>
88 class Dense
89  : public EnableLinOp<Dense<ValueType>>,
90  public ConvertibleTo<Dense<next_precision<ValueType>>>,
91 #if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16
92  public ConvertibleTo<Dense<next_precision<ValueType, 2>>>,
93 #endif
94 #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16
95  public ConvertibleTo<Dense<next_precision<ValueType, 3>>>,
96 #endif
97  public ConvertibleTo<Coo<ValueType, int32>>,
98  public ConvertibleTo<Coo<ValueType, int64>>,
99  public ConvertibleTo<Csr<ValueType, int32>>,
100  public ConvertibleTo<Csr<ValueType, int64>>,
101  public ConvertibleTo<Ell<ValueType, int32>>,
102  public ConvertibleTo<Ell<ValueType, int64>>,
103  public ConvertibleTo<Fbcsr<ValueType, int32>>,
104  public ConvertibleTo<Fbcsr<ValueType, int64>>,
105  public ConvertibleTo<Hybrid<ValueType, int32>>,
106  public ConvertibleTo<Hybrid<ValueType, int64>>,
107  public ConvertibleTo<Sellp<ValueType, int32>>,
108  public ConvertibleTo<Sellp<ValueType, int64>>,
109  public ConvertibleTo<SparsityCsr<ValueType, int32>>,
110  public ConvertibleTo<SparsityCsr<ValueType, int64>>,
111  public DiagonalExtractable<ValueType>,
112  public ReadableFromMatrixData<ValueType, int32>,
113  public ReadableFromMatrixData<ValueType, int64>,
114  public WritableToMatrixData<ValueType, int32>,
115  public WritableToMatrixData<ValueType, int64>,
116  public Transposable,
117  public Permutable<int32>,
118  public Permutable<int64>,
119  public EnableAbsoluteComputation<remove_complex<Dense<ValueType>>>,
120  public ScaledIdentityAddable {
121  friend class EnablePolymorphicObject<Dense, LinOp>;
122  friend class Coo<ValueType, int32>;
123  friend class Coo<ValueType, int64>;
124  friend class Csr<ValueType, int32>;
125  friend class Csr<ValueType, int64>;
126  friend class Diagonal<ValueType>;
127  friend class Ell<ValueType, int32>;
128  friend class Ell<ValueType, int64>;
129  friend class Fbcsr<ValueType, int32>;
130  friend class Fbcsr<ValueType, int64>;
131  friend class Hybrid<ValueType, int32>;
132  friend class Hybrid<ValueType, int64>;
133  friend class Sellp<ValueType, int32>;
134  friend class Sellp<ValueType, int64>;
135  friend class SparsityCsr<ValueType, int32>;
136  friend class SparsityCsr<ValueType, int64>;
137  friend class Dense<to_complex<ValueType>>;
138  friend class experimental::distributed::Vector<ValueType>;
139  friend class experimental::distributed::detail::VectorCache<ValueType>;
140  GKO_ASSERT_SUPPORTED_VALUE_TYPE;
141 
142 public:
145  using ConvertibleTo<Dense<next_precision<ValueType>>>::convert_to;
146  using ConvertibleTo<Dense<next_precision<ValueType>>>::move_to;
147  using ConvertibleTo<Coo<ValueType, int32>>::convert_to;
148  using ConvertibleTo<Coo<ValueType, int32>>::move_to;
149  using ConvertibleTo<Coo<ValueType, int64>>::convert_to;
150  using ConvertibleTo<Coo<ValueType, int64>>::move_to;
151  using ConvertibleTo<Csr<ValueType, int32>>::convert_to;
152  using ConvertibleTo<Csr<ValueType, int32>>::move_to;
153  using ConvertibleTo<Csr<ValueType, int64>>::convert_to;
154  using ConvertibleTo<Csr<ValueType, int64>>::move_to;
155  using ConvertibleTo<Ell<ValueType, int32>>::convert_to;
156  using ConvertibleTo<Ell<ValueType, int32>>::move_to;
157  using ConvertibleTo<Ell<ValueType, int64>>::convert_to;
158  using ConvertibleTo<Ell<ValueType, int64>>::move_to;
159  using ConvertibleTo<Fbcsr<ValueType, int32>>::convert_to;
160  using ConvertibleTo<Fbcsr<ValueType, int32>>::move_to;
161  using ConvertibleTo<Fbcsr<ValueType, int64>>::convert_to;
162  using ConvertibleTo<Fbcsr<ValueType, int64>>::move_to;
163  using ConvertibleTo<Hybrid<ValueType, int32>>::convert_to;
164  using ConvertibleTo<Hybrid<ValueType, int32>>::move_to;
165  using ConvertibleTo<Hybrid<ValueType, int64>>::convert_to;
166  using ConvertibleTo<Hybrid<ValueType, int64>>::move_to;
167  using ConvertibleTo<Sellp<ValueType, int32>>::convert_to;
168  using ConvertibleTo<Sellp<ValueType, int32>>::move_to;
169  using ConvertibleTo<Sellp<ValueType, int64>>::convert_to;
170  using ConvertibleTo<Sellp<ValueType, int64>>::move_to;
171  using ConvertibleTo<SparsityCsr<ValueType, int32>>::convert_to;
172  using ConvertibleTo<SparsityCsr<ValueType, int32>>::move_to;
173  using ConvertibleTo<SparsityCsr<ValueType, int64>>::convert_to;
174  using ConvertibleTo<SparsityCsr<ValueType, int64>>::move_to;
177 
178  using value_type = ValueType;
179  using index_type = int64;
180  using transposed_type = Dense<ValueType>;
181  using mat_data = matrix_data<ValueType, int64>;
182  using mat_data32 = matrix_data<ValueType, int32>;
183  using device_mat_data = device_matrix_data<ValueType, int64>;
184  using device_mat_data32 = device_matrix_data<ValueType, int32>;
185  using absolute_type = remove_complex<Dense>;
186  using real_type = absolute_type;
187  using complex_type = to_complex<Dense>;
188 
189  using row_major_range = gko::range<gko::accessor::row_major<ValueType, 2>>;
190 
197  static std::unique_ptr<Dense> create_with_config_of(
199  {
200  // De-referencing `other` before calling the functions (instead of
201  // using operator `->`) is currently required to be compatible with
202  // CUDA 10.1.
203  // Otherwise, it results in a compile error.
204  return (*other).create_with_same_config();
205  }
206 
218  static std::unique_ptr<Dense> create_with_type_of(
219  ptr_param<const Dense> other, std::shared_ptr<const Executor> exec,
220  const dim<2>& size = dim<2>{})
221  {
222  // See create_with_config_of()
223  return (*other).create_with_type_of_impl(exec, size, size[1]);
224  }
225 
234  static std::unique_ptr<Dense> create_with_type_of(
235  ptr_param<const Dense> other, std::shared_ptr<const Executor> exec,
236  const dim<2>& size, size_type stride)
237  {
238  // See create_with_config_of()
239  return (*other).create_with_type_of_impl(exec, size, stride);
240  }
241 
252  static std::unique_ptr<Dense> create_with_type_of(
253  ptr_param<const Dense> other, std::shared_ptr<const Executor> exec,
254  const dim<2>& size, const dim<2>& local_size, size_type stride)
255  {
256  // See create_with_config_of()
257  return (*other).create_with_type_of_impl(exec, size, stride);
258  }
259 
268  static std::unique_ptr<Dense> create_view_of(ptr_param<Dense> other)
269  {
270  return other->create_view_of_impl();
271  }
272 
280  static std::unique_ptr<const Dense> create_const_view_of(
282  {
283  return other->create_const_view_of_impl();
284  }
285 
286  friend class Dense<previous_precision<ValueType>>;
287 
288  void convert_to(Dense<next_precision<ValueType>>* result) const override;
289 
290  void move_to(Dense<next_precision<ValueType>>* result) override;
291 
292 #if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16
293  friend class Dense<previous_precision<ValueType, 2>>;
296 
297  void convert_to(Dense<next_precision<ValueType, 2>>* result) const override;
298 
299  void move_to(Dense<next_precision<ValueType, 2>>* result) override;
300 #endif
301 
302 #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16
303  friend class Dense<previous_precision<ValueType, 3>>;
306 
307  void convert_to(Dense<next_precision<ValueType, 3>>* result) const override;
308 
309  void move_to(Dense<next_precision<ValueType, 3>>* result) override;
310 #endif
311 
312  void convert_to(Coo<ValueType, int32>* result) const override;
313 
314  void move_to(Coo<ValueType, int32>* result) override;
315 
316  void convert_to(Coo<ValueType, int64>* result) const override;
317 
318  void move_to(Coo<ValueType, int64>* result) override;
319 
320  void convert_to(Csr<ValueType, int32>* result) const override;
321 
322  void move_to(Csr<ValueType, int32>* result) override;
323 
324  void convert_to(Csr<ValueType, int64>* result) const override;
325 
326  void move_to(Csr<ValueType, int64>* result) override;
327 
328  void convert_to(Ell<ValueType, int32>* result) const override;
329 
330  void move_to(Ell<ValueType, int32>* result) override;
331 
332  void convert_to(Ell<ValueType, int64>* result) const override;
333 
334  void move_to(Ell<ValueType, int64>* result) override;
335 
336  void convert_to(Fbcsr<ValueType, int32>* result) const override;
337 
338  void move_to(Fbcsr<ValueType, int32>* result) override;
339 
340  void convert_to(Fbcsr<ValueType, int64>* result) const override;
341 
342  void move_to(Fbcsr<ValueType, int64>* result) override;
343 
344  void convert_to(Hybrid<ValueType, int32>* result) const override;
345 
346  void move_to(Hybrid<ValueType, int32>* result) override;
347 
348  void convert_to(Hybrid<ValueType, int64>* result) const override;
349 
350  void move_to(Hybrid<ValueType, int64>* result) override;
351 
352  void convert_to(Sellp<ValueType, int32>* result) const override;
353 
354  void move_to(Sellp<ValueType, int32>* result) override;
355 
356  void convert_to(Sellp<ValueType, int64>* result) const override;
357 
358  void move_to(Sellp<ValueType, int64>* result) override;
359 
360  void convert_to(SparsityCsr<ValueType, int32>* result) const override;
361 
362  void move_to(SparsityCsr<ValueType, int32>* result) override;
363 
364  void convert_to(SparsityCsr<ValueType, int64>* result) const override;
365 
366  void move_to(SparsityCsr<ValueType, int64>* result) override;
367 
368  void read(const mat_data& data) override;
369 
370  void read(const mat_data32& data) override;
371 
372  void read(const device_mat_data& data) override;
373 
374  void read(const device_mat_data32& data) override;
375 
376  void read(device_mat_data&& data) override;
377 
378  void read(device_mat_data32&& data) override;
379 
380  void write(mat_data& data) const override;
381 
382  void write(mat_data32& data) const override;
383 
384  std::unique_ptr<LinOp> transpose() const override;
385 
386  std::unique_ptr<LinOp> conj_transpose() const override;
387 
394  void transpose(ptr_param<Dense> output) const;
395 
402  void conj_transpose(ptr_param<Dense> output) const;
403 
409  void fill(const ValueType value);
410 
425  std::unique_ptr<Dense> permute(
426  ptr_param<const Permutation<int32>> permutation,
428 
432  std::unique_ptr<Dense> permute(
433  ptr_param<const Permutation<int64>> permutation,
435 
441  void permute(ptr_param<const Permutation<int32>> permutation,
442  ptr_param<Dense> output, permute_mode mode) const;
443 
448  void permute(ptr_param<const Permutation<int64>> permutation,
449  ptr_param<Dense> output, permute_mode mode) const;
450 
464  std::unique_ptr<Dense> permute(
465  ptr_param<const Permutation<int32>> row_permutation,
466  ptr_param<const Permutation<int32>> column_permutation,
467  bool invert = false) const;
468 
473  std::unique_ptr<Dense> permute(
474  ptr_param<const Permutation<int64>> row_permutation,
475  ptr_param<const Permutation<int64>> column_permutation,
476  bool invert = false) const;
477 
484  void permute(ptr_param<const Permutation<int32>> row_permutation,
485  ptr_param<const Permutation<int32>> column_permutation,
486  ptr_param<Dense> output, bool invert = false) const;
487 
492  void permute(ptr_param<const Permutation<int64>> row_permutation,
493  ptr_param<const Permutation<int64>> column_permutation,
494  ptr_param<Dense> output, bool invert = false) const;
495 
505  std::unique_ptr<Dense> scale_permute(
508 
513  std::unique_ptr<Dense> scale_permute(
516 
523  void scale_permute(
525  ptr_param<Dense> output, permute_mode mode) const;
526 
531  void scale_permute(
533  ptr_param<Dense> output, permute_mode mode) const;
534 
547  std::unique_ptr<Dense> scale_permute(
548  ptr_param<const ScaledPermutation<value_type, int32>> row_permutation,
550  column_permutation,
551  bool invert = false) const;
552 
557  std::unique_ptr<Dense> scale_permute(
558  ptr_param<const ScaledPermutation<value_type, int64>> row_permutation,
560  column_permutation,
561  bool invert = false) const;
562 
569  void scale_permute(
570  ptr_param<const ScaledPermutation<value_type, int32>> row_permutation,
572  column_permutation,
573  ptr_param<Dense> output, bool invert = false) const;
574 
580  void scale_permute(
581  ptr_param<const ScaledPermutation<value_type, int64>> row_permutation,
583  column_permutation,
584  ptr_param<Dense> output, bool invert = false) const;
585 
586  std::unique_ptr<LinOp> permute(
587  const array<int32>* permutation_indices) const override;
588 
589  std::unique_ptr<LinOp> permute(
590  const array<int64>* permutation_indices) const override;
591 
601  void permute(const array<int32>* permutation_indices,
602  ptr_param<Dense> output) const;
603 
607  void permute(const array<int64>* permutation_indices,
608  ptr_param<Dense> output) const;
609 
610  std::unique_ptr<LinOp> inverse_permute(
611  const array<int32>* permutation_indices) const override;
612 
613  std::unique_ptr<LinOp> inverse_permute(
614  const array<int64>* permutation_indices) const override;
615 
626  void inverse_permute(const array<int32>* permutation_indices,
627  ptr_param<Dense> output) const;
628 
632  void inverse_permute(const array<int64>* permutation_indices,
633  ptr_param<Dense> output) const;
634 
635  std::unique_ptr<LinOp> row_permute(
636  const array<int32>* permutation_indices) const override;
637 
638  std::unique_ptr<LinOp> row_permute(
639  const array<int64>* permutation_indices) const override;
640 
650  void row_permute(const array<int32>* permutation_indices,
651  ptr_param<Dense> output) const;
652 
656  void row_permute(const array<int64>* permutation_indices,
657  ptr_param<Dense> output) const;
658 
669  std::unique_ptr<Dense> row_gather(const array<int32>* gather_indices) const;
670 
674  std::unique_ptr<Dense> row_gather(const array<int64>* gather_indices) const;
675 
688  void row_gather(const array<int32>* gather_indices,
689  ptr_param<LinOp> row_collection) const;
690 
694  void row_gather(const array<int64>* gather_indices,
695  ptr_param<LinOp> row_collection) const;
696 
711  const array<int32>* gather_indices,
713  ptr_param<LinOp> row_collection) const;
714 
720  const array<int64>* gather_indices,
722  ptr_param<LinOp> row_collection) const;
723 
724  std::unique_ptr<LinOp> column_permute(
725  const array<int32>* permutation_indices) const override;
726 
727  std::unique_ptr<LinOp> column_permute(
728  const array<int64>* permutation_indices) const override;
729 
739  void column_permute(const array<int32>* permutation_indices,
740  ptr_param<Dense> output) const;
741 
745  void column_permute(const array<int64>* permutation_indices,
746  ptr_param<Dense> output) const;
747 
748  std::unique_ptr<LinOp> inverse_row_permute(
749  const array<int32>* permutation_indices) const override;
750 
751  std::unique_ptr<LinOp> inverse_row_permute(
752  const array<int64>* permutation_indices) const override;
753 
763  void inverse_row_permute(const array<int32>* permutation_indices,
764  ptr_param<Dense> output) const;
765 
769  void inverse_row_permute(const array<int64>* permutation_indices,
770  ptr_param<Dense> output) const;
771 
772  std::unique_ptr<LinOp> inverse_column_permute(
773  const array<int32>* permutation_indices) const override;
774 
775  std::unique_ptr<LinOp> inverse_column_permute(
776  const array<int64>* permutation_indices) const override;
777 
787  void inverse_column_permute(const array<int32>* permutation_indices,
788  ptr_param<Dense> output) const;
789 
793  void inverse_column_permute(const array<int64>* permutation_indices,
794  ptr_param<Dense> output) const;
795 
796  std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
797 
805  void extract_diagonal(ptr_param<Diagonal<ValueType>> output) const;
806 
807  std::unique_ptr<absolute_type> compute_absolute() const override;
808 
816  void compute_absolute(ptr_param<absolute_type> output) const;
817 
818  void compute_absolute_inplace() override;
819 
824  std::unique_ptr<complex_type> make_complex() const;
825 
831  void make_complex(ptr_param<complex_type> result) const;
832 
837  std::unique_ptr<real_type> get_real() const;
838 
842  void get_real(ptr_param<real_type> result) const;
843 
848  std::unique_ptr<real_type> get_imag() const;
849 
854  void get_imag(ptr_param<real_type> result) const;
855 
861  value_type* get_values() noexcept { return values_.get_data(); }
862 
870  const value_type* get_const_values() const noexcept
871  {
872  return values_.get_const_data();
873  }
874 
880  size_type get_stride() const noexcept { return stride_; }
881 
888  {
889  return values_.get_size();
890  }
891 
902  value_type& at(size_type row, size_type col) noexcept
903  {
904  return values_.get_data()[linearize_index(row, col)];
905  }
906 
910  value_type at(size_type row, size_type col) const noexcept
911  {
912  return values_.get_const_data()[linearize_index(row, col)];
913  }
914 
929  ValueType& at(size_type idx) noexcept
930  {
931  return values_.get_data()[linearize_index(idx)];
932  }
933 
937  ValueType at(size_type idx) const noexcept
938  {
939  return values_.get_const_data()[linearize_index(idx)];
940  }
941 
951  void scale(ptr_param<const LinOp> alpha);
952 
962  void inv_scale(ptr_param<const LinOp> alpha);
963 
975 
987 
997 
1010  array<char>& tmp) const;
1011 
1021  ptr_param<LinOp> result) const;
1022 
1035  array<char>& tmp) const;
1036 
1044  void compute_norm2(ptr_param<LinOp> result) const;
1045 
1056  void compute_norm2(ptr_param<LinOp> result, array<char>& tmp) const;
1057 
1065  void compute_norm1(ptr_param<LinOp> result) const;
1066 
1077  void compute_norm1(ptr_param<LinOp> result, array<char>& tmp) const;
1078 
1087  void compute_squared_norm2(ptr_param<LinOp> result) const;
1088 
1100  void compute_squared_norm2(ptr_param<LinOp> result, array<char>& tmp) const;
1101 
1109  void compute_mean(ptr_param<LinOp> result) const;
1110 
1121  void compute_mean(ptr_param<LinOp> result, array<char>& tmp) const;
1122 
1133  std::unique_ptr<Dense> create_submatrix(const span& rows,
1134  const span& columns,
1135  const size_type stride)
1136  {
1137  return this->create_submatrix_impl(rows, columns, stride);
1138  }
1139 
1146  std::unique_ptr<Dense> create_submatrix(const span& rows,
1147  const span& columns)
1148  {
1149  return create_submatrix(rows, columns, this->get_stride());
1150  }
1151 
1152 
1161  std::unique_ptr<Dense> create_submatrix(const local_span& rows,
1162  const local_span& columns,
1163  dim<2> size)
1164  {
1165  dim<2> deduced_size{rows.length(), columns.length()};
1166  GKO_ASSERT_EQUAL_DIMENSIONS(deduced_size, size);
1167  return create_submatrix(rows, columns, this->get_stride());
1168  }
1169 
1177  std::unique_ptr<real_type> create_real_view();
1178 
1182  std::unique_ptr<const real_type> create_real_view() const;
1183 
1196  static std::unique_ptr<Dense> create(std::shared_ptr<const Executor> exec,
1197  const dim<2>& size = {},
1198  size_type stride = 0);
1199 
1216  static std::unique_ptr<Dense> create(std::shared_ptr<const Executor> exec,
1217  const dim<2>& size,
1218  array<value_type> values,
1219  size_type stride);
1220 
1225  template <typename InputValueType>
1226  GKO_DEPRECATED(
1227  "explicitly construct the gko::array argument instead of passing an"
1228  "initializer list")
1229  static std::unique_ptr<Dense> create(
1230  std::shared_ptr<const Executor> exec, const dim<2>& size,
1231  std::initializer_list<InputValueType> values, size_type stride)
1232  {
1233  return create(exec, size, array<value_type>{exec, std::move(values)},
1234  stride);
1235  }
1236 
1248  static std::unique_ptr<const Dense> create_const(
1249  std::shared_ptr<const Executor> exec, const dim<2>& size,
1250  gko::detail::const_array_view<ValueType>&& values, size_type stride);
1251 
1257  Dense& operator=(const Dense&);
1258 
1264  Dense& operator=(Dense&&);
1265 
1270  Dense(const Dense&);
1271 
1276  Dense(Dense&&);
1277 
1278 protected:
1279  Dense(std::shared_ptr<const Executor> exec, const dim<2>& size = {},
1280  size_type stride = 0);
1281 
1282  Dense(std::shared_ptr<const Executor> exec, const dim<2>& size,
1283  array<value_type> values, size_type stride);
1284 
1291  virtual std::unique_ptr<Dense> create_with_same_config() const
1292  {
1293  return Dense::create(this->get_executor(), this->get_size(),
1294  this->get_stride());
1295  }
1296 
1304  virtual std::unique_ptr<Dense> create_with_type_of_impl(
1305  std::shared_ptr<const Executor> exec, const dim<2>& size,
1306  size_type stride) const
1307  {
1308  return Dense::create(exec, size, stride);
1309  }
1310 
1317  virtual std::unique_ptr<Dense> create_view_of_impl()
1318  {
1319  auto exec = this->get_executor();
1320  return Dense::create(
1321  exec, this->get_size(),
1323  this->get_values()),
1324  this->get_stride());
1325  }
1326 
1333  virtual std::unique_ptr<const Dense> create_const_view_of_impl() const
1334  {
1335  auto exec = this->get_executor();
1336  return Dense::create_const(
1337  exec, this->get_size(),
1339  this->get_const_values()),
1340  this->get_stride());
1341  }
1342 
1343  template <typename IndexType>
1344  void convert_impl(Coo<ValueType, IndexType>* result) const;
1345 
1346  template <typename IndexType>
1347  void convert_impl(Csr<ValueType, IndexType>* result) const;
1348 
1349  template <typename IndexType>
1350  void convert_impl(Ell<ValueType, IndexType>* result) const;
1351 
1352  template <typename IndexType>
1353  void convert_impl(Fbcsr<ValueType, IndexType>* result) const;
1354 
1355  template <typename IndexType>
1356  void convert_impl(Hybrid<ValueType, IndexType>* result) const;
1357 
1358  template <typename IndexType>
1359  void convert_impl(Sellp<ValueType, IndexType>* result) const;
1360 
1361  template <typename IndexType>
1362  void convert_impl(SparsityCsr<ValueType, IndexType>* result) const;
1363 
1370  virtual void scale_impl(const LinOp* alpha);
1371 
1378  virtual void inv_scale_impl(const LinOp* alpha);
1379 
1386  virtual void add_scaled_impl(const LinOp* alpha, const LinOp* b);
1387 
1394  virtual void sub_scaled_impl(const LinOp* alpha, const LinOp* b);
1395 
1402  virtual void compute_dot_impl(const LinOp* b, LinOp* result) const;
1403 
1410  virtual void compute_conj_dot_impl(const LinOp* b, LinOp* result) const;
1411 
1418  virtual void compute_norm2_impl(LinOp* result) const;
1419 
1426  virtual void compute_norm1_impl(LinOp* result) const;
1427 
1434  virtual void compute_squared_norm2_impl(LinOp* result) const;
1435 
1439  virtual void compute_mean_impl(LinOp* result) const;
1440 
1449  void resize(gko::dim<2> new_size);
1450 
1458  virtual std::unique_ptr<Dense> create_submatrix_impl(
1459  const span& rows, const span& columns, const size_type stride);
1460 
1461  void apply_impl(const LinOp* b, LinOp* x) const override;
1462 
1463  void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta,
1464  LinOp* x) const override;
1465 
1466  size_type linearize_index(size_type row, size_type col) const noexcept
1467  {
1468  return row * stride_ + col;
1469  }
1470 
1471  size_type linearize_index(size_type idx) const noexcept
1472  {
1473  return linearize_index(idx / this->get_size()[1],
1474  idx % this->get_size()[1]);
1475  }
1476 
1477  template <typename IndexType>
1478  void permute_impl(const Permutation<IndexType>* permutation,
1479  permute_mode mode, Dense* output) const;
1480 
1481  template <typename IndexType>
1482  void permute_impl(const Permutation<IndexType>* row_permutation,
1483  const Permutation<IndexType>* col_permutation,
1484  bool invert, Dense* output) const;
1485 
1486  template <typename IndexType>
1487  void scale_permute_impl(
1488  const ScaledPermutation<ValueType, IndexType>* permutation,
1489  permute_mode mode, Dense* output) const;
1490 
1491  template <typename IndexType>
1492  void scale_permute_impl(
1493  const ScaledPermutation<ValueType, IndexType>* row_permutation,
1494  const ScaledPermutation<ValueType, IndexType>* column_permutation,
1495  bool invert, Dense* output) const;
1496 
1497  template <typename OutputType, typename IndexType>
1498  void row_gather_impl(const array<IndexType>* row_idxs,
1499  Dense<OutputType>* row_collection) const;
1500 
1501  template <typename OutputType, typename IndexType>
1502  void row_gather_impl(const Dense<ValueType>* alpha,
1503  const array<IndexType>* row_idxs,
1504  const Dense<ValueType>* beta,
1505  Dense<OutputType>* row_collection) const;
1506 
1507 private:
1508  size_type stride_;
1509  array<value_type> values_;
1510 
1511  void add_scaled_identity_impl(const LinOp* a, const LinOp* b) override;
1512 };
1513 
1514 
1515 } // namespace matrix
1516 
1517 
1518 namespace detail {
1519 
1520 
1521 template <typename ValueType>
1522 struct temporary_clone_helper<matrix::Dense<ValueType>> {
1523  static std::unique_ptr<matrix::Dense<ValueType>> create(
1524  std::shared_ptr<const Executor> exec, matrix::Dense<ValueType>* ptr,
1525  bool copy_data)
1526  {
1527  if (copy_data) {
1528  return gko::clone(std::move(exec), ptr);
1529  } else {
1530  return matrix::Dense<ValueType>::create(exec, ptr->get_size());
1531  }
1532  }
1533 };
1534 
1535 
1536 } // namespace detail
1537 
1538 
1546 template <typename VecPtr>
1547 std::unique_ptr<matrix::Dense<typename detail::pointee<VecPtr>::value_type>>
1548 make_dense_view(VecPtr&& vector)
1549 {
1550  using value_type = typename detail::pointee<VecPtr>::value_type;
1552 }
1553 
1554 
1562 template <typename VecPtr>
1563 std::unique_ptr<
1564  const matrix::Dense<typename detail::pointee<VecPtr>::value_type>>
1565 make_const_dense_view(VecPtr&& vector)
1566 {
1567  using value_type = typename detail::pointee<VecPtr>::value_type;
1569 }
1570 
1571 
1592 template <typename Matrix, typename... TArgs>
1593 std::unique_ptr<Matrix> initialize(
1594  size_type stride, std::initializer_list<typename Matrix::value_type> vals,
1595  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1596 {
1598  size_type num_rows = vals.size();
1599  auto tmp = dense::create(exec->get_master(), dim<2>{num_rows, 1}, stride);
1600  size_type idx = 0;
1601  for (const auto& elem : vals) {
1602  tmp->at(idx) = elem;
1603  ++idx;
1604  }
1605  auto mtx = Matrix::create(exec, std::forward<TArgs>(create_args)...);
1606  tmp->move_to(mtx);
1607  return mtx;
1608 }
1609 
1630 template <typename Matrix, typename... TArgs>
1631 std::unique_ptr<Matrix> initialize(
1632  std::initializer_list<typename Matrix::value_type> vals,
1633  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1634 {
1635  return initialize<Matrix>(1, vals, std::move(exec),
1636  std::forward<TArgs>(create_args)...);
1637 }
1638 
1639 
1660 template <typename Matrix, typename... TArgs>
1661 std::unique_ptr<Matrix> initialize(
1662  size_type stride,
1663  std::initializer_list<std::initializer_list<typename Matrix::value_type>>
1664  vals,
1665  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1666 {
1668  size_type num_rows = vals.size();
1669  size_type num_cols = num_rows > 0 ? begin(vals)->size() : 1;
1670  auto tmp =
1671  dense::create(exec->get_master(), dim<2>{num_rows, num_cols}, stride);
1672  size_type ridx = 0;
1673  for (const auto& row : vals) {
1674  size_type cidx = 0;
1675  for (const auto& elem : row) {
1676  tmp->at(ridx, cidx) = elem;
1677  ++cidx;
1678  }
1679  ++ridx;
1680  }
1681  auto mtx = Matrix::create(exec, std::forward<TArgs>(create_args)...);
1682  tmp->move_to(mtx);
1683  return mtx;
1684 }
1685 
1686 
1708 template <typename Matrix, typename... TArgs>
1709 std::unique_ptr<Matrix> initialize(
1710  std::initializer_list<std::initializer_list<typename Matrix::value_type>>
1711  vals,
1712  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1713 {
1714  return initialize<Matrix>(vals.size() > 0 ? begin(vals)->size() : 0, vals,
1715  std::move(exec),
1716  std::forward<TArgs>(create_args)...);
1717 }
1718 
1719 
1720 } // namespace gko
1721 
1722 
1723 #endif // GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
gko::matrix::Dense::permute
std::unique_ptr< Dense > permute(ptr_param< const Permutation< int32 >> permutation, permute_mode mode=permute_mode::symmetric) const
Creates a permuted copy of this matrix with the given permutation .
gko::matrix::Dense::inverse_column_permute
std::unique_ptr< LinOp > inverse_column_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the row permutation of the inverse permuted object.
gko::matrix::Dense::add_scaled
void add_scaled(ptr_param< const LinOp > alpha, ptr_param< const LinOp > b)
Adds b scaled by alpha to the matrix (aka: BLAS axpy).
gko::matrix::Fbcsr
Fixed-block compressed sparse row storage matrix format.
Definition: csr.hpp:46
gko::EnablePolymorphicAssignment< ConcreteLinOp >::move_to
void move_to(result_type *result) override
Definition: polymorphic_object.hpp:751
gko::EnablePolymorphicAssignment< ConcreteLinOp >::convert_to
void convert_to(result_type *result) const override
Definition: polymorphic_object.hpp:749
gko::matrix::Csr
CSR is a matrix format which stores only the nonzero coefficients by compressing each row of the matr...
Definition: matrix.hpp:30
gko::matrix::permute_mode::rows
The rows will be permuted.
gko::matrix::Dense::transpose
std::unique_ptr< LinOp > transpose() const override
Returns a LinOp representing the transpose of the Transposable object.
gko::matrix::permute_mode::columns
The columns will be permuted.
gko::ReadableFromMatrixData< ValueType, int32 >::read
virtual void read(const matrix_data< ValueType, int32 > &data)=0
Reads a matrix from a matrix_data structure.
gko::matrix::Dense
Dense is a matrix format which explicitly stores all values of the matrix.
Definition: dense_cache.hpp:28
gko::matrix::SparsityCsr
SparsityCsr is a matrix format which stores only the sparsity pattern of a sparse matrix by compressi...
Definition: csr.hpp:40
gko::matrix::Dense::at
ValueType at(size_type idx) const noexcept
Returns a single element of the matrix.
Definition: dense.hpp:937
gko::matrix::Dense::row_gather
std::unique_ptr< Dense > row_gather(const array< int32 > *gather_indices) const
Create a Dense matrix consisting of the given rows from this matrix.
gko::matrix::Dense::make_complex
std::unique_ptr< complex_type > make_complex() const
Creates a complex copy of the original matrix.
gko::matrix::Dense::create
static std::unique_ptr< Dense > create(std::shared_ptr< const Executor > exec, const dim< 2 > &size={}, size_type stride=0)
Creates an uninitialized Dense matrix of the specified size.
gko::matrix::Dense::create_with_type_of
static std::unique_ptr< Dense > create_with_type_of(ptr_param< const Dense > other, std::shared_ptr< const Executor > exec, const dim< 2 > &size, size_type stride)
Definition: dense.hpp:234
gko::matrix::Dense::scale
void scale(ptr_param< const LinOp > alpha)
Scales the matrix with a scalar (aka: BLAS scal).
gko::matrix::ScaledPermutation
ScaledPermutation is a matrix combining a permutation with scaling factors.
Definition: scaled_permutation.hpp:36
gko::size_type
std::size_t size_type
Integral type used for allocation quantities.
Definition: types.hpp:90
gko::matrix::Dense::get_real
std::unique_ptr< real_type > get_real() const
Creates a new real matrix and extracts the real part of the original matrix into that.
gko::matrix::Dense::inv_scale
void inv_scale(ptr_param< const LinOp > alpha)
Scales the matrix with the inverse of a scalar.
gko::matrix::Permutation
Permutation is a matrix format that represents a permutation matrix, i.e.
Definition: csr.hpp:52
gko::matrix::Dense::fill
void fill(const ValueType value)
Fill the dense matrix with a given value.
gko::make_const_array_view
detail::const_array_view< ValueType > make_const_array_view(std::shared_ptr< const Executor > exec, size_type size, const ValueType *data)
Helper function to create a const array view deducing the value type.
Definition: array.hpp:820
gko::matrix::Dense::compute_dot
void compute_dot(ptr_param< const LinOp > b, ptr_param< LinOp > result) const
Computes the column-wise dot product of this matrix and b.
gko::matrix::Dense::create_submatrix
std::unique_ptr< Dense > create_submatrix(const span &rows, const span &columns, const size_type stride)
Create a submatrix from the original matrix.
Definition: dense.hpp:1133
gko::matrix::Dense::create_with_config_of
static std::unique_ptr< Dense > create_with_config_of(ptr_param< const Dense > other)
Creates a Dense matrix with the same size and stride as another Dense matrix.
Definition: dense.hpp:197
gko::initialize
std::unique_ptr< Matrix > initialize(size_type stride, std::initializer_list< typename Matrix::value_type > vals, std::shared_ptr< const Executor > exec, TArgs &&... create_args)
Creates and initializes a column-vector.
Definition: dense.hpp:1593
gko::matrix::Dense::at
value_type & at(size_type row, size_type col) noexcept
Returns a single element of the matrix.
Definition: dense.hpp:902
gko::clone
detail::cloned_type< Pointer > clone(const Pointer &p)
Creates a unique clone of the object pointed to by p.
Definition: utils_helper.hpp:173
gko::matrix::Dense::get_stride
size_type get_stride() const noexcept
Returns the stride of the matrix.
Definition: dense.hpp:880
gko::matrix::Dense::create_view_of
static std::unique_ptr< Dense > create_view_of(ptr_param< Dense > other)
Creates a Dense matrix, where the underlying array is a view of another Dense matrix' array.
Definition: dense.hpp:268
gko::range
A range is a multidimensional view of the memory.
Definition: range.hpp:304
gko
The Ginkgo namespace.
Definition: abstract_factory.hpp:20
gko::matrix::Dense::get_num_stored_elements
size_type get_num_stored_elements() const noexcept
Returns the number of elements explicitly stored in the matrix.
Definition: dense.hpp:887
gko::matrix::Dense::sub_scaled
void sub_scaled(ptr_param< const LinOp > alpha, ptr_param< const LinOp > b)
Subtracts b scaled by alpha from the matrix (aka: BLAS axpy).
gko::matrix::Dense::get_values
value_type * get_values() noexcept
Returns a pointer to the array of values of the matrix.
Definition: dense.hpp:861
gko::array
An array is a container which encapsulates fixed-sized arrays, stored on the Executor tied to the arr...
Definition: array.hpp:26
gko::matrix::Dense::compute_norm2
void compute_norm2(ptr_param< LinOp > result) const
Computes the column-wise Euclidean (L^2) norm of this matrix.
gko::matrix::Dense::row_permute
std::unique_ptr< LinOp > row_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the row permutation of the Permutable object.
gko::matrix::Dense::create_with_type_of
static std::unique_ptr< Dense > create_with_type_of(ptr_param< const Dense > other, std::shared_ptr< const Executor > exec, const dim< 2 > &size=dim< 2 >{})
Creates a Dense matrix with the same type as another Dense matrix but on a different executor and wit...
Definition: dense.hpp:218
gko::span
A span is a lightweight structure used to create sub-ranges from other ranges.
Definition: range.hpp:46
gko::dim< 2 >
gko::matrix::Diagonal
This class is a utility which efficiently implements the diagonal matrix (a linear operator which sca...
Definition: lin_op.hpp:31
gko::matrix::Dense::create_const
static std::unique_ptr< const Dense > create_const(std::shared_ptr< const Executor > exec, const dim< 2 > &size, gko::detail::const_array_view< ValueType > &&values, size_type stride)
Creates a constant (immutable) Dense matrix from a constant array.
gko::ptr_param
This class is used for function parameters in the place of raw pointers.
Definition: utils_helper.hpp:41
gko::matrix::Dense::at
value_type at(size_type row, size_type col) const noexcept
Returns a single element of the matrix.
Definition: dense.hpp:910
gko::array::get_data
value_type * get_data() noexcept
Returns a pointer to the block of memory used to store the elements of the array.
Definition: array.hpp:687
gko::matrix::Dense::compute_mean
void compute_mean(ptr_param< LinOp > result) const
Computes the column-wise arithmetic mean of this matrix.
gko::matrix::Dense::inverse_row_permute
std::unique_ptr< LinOp > inverse_row_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the row permutation of the inverse permuted object.
gko::matrix::permute_mode::symmetric
The rows and columns will be permuted.
gko::matrix::Dense::get_imag
std::unique_ptr< real_type > get_imag() const
Creates a new real matrix and extracts the imaginary part of the original matrix into that.
gko::matrix::Dense::compute_squared_norm2
void compute_squared_norm2(ptr_param< LinOp > result) const
Computes the square of the column-wise Euclidean (L^2) norm of this matrix.
gko::matrix::Dense::operator=
Dense & operator=(const Dense &)
Copy-assigns a Dense matrix.
gko::matrix::Dense::inverse_permute
std::unique_ptr< LinOp > inverse_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the symmetric inverse row and column permutation of the Permutable objec...
gko::matrix::Dense::create_with_type_of
static std::unique_ptr< Dense > create_with_type_of(ptr_param< const Dense > other, std::shared_ptr< const Executor > exec, const dim< 2 > &size, const dim< 2 > &local_size, size_type stride)
Definition: dense.hpp:252
gko::stop::mode
mode
The mode for the residual norm criterion.
Definition: residual_norm.hpp:37
gko::matrix::Dense::scale_permute
std::unique_ptr< Dense > scale_permute(ptr_param< const ScaledPermutation< value_type, int32 >> permutation, permute_mode mode=permute_mode::symmetric) const
Creates a scaled and permuted copy of this matrix.
gko::matrix::Dense::compute_norm1
void compute_norm1(ptr_param< LinOp > result) const
Computes the column-wise (L^1) norm of this matrix.
gko::matrix::Dense::get_const_values
const value_type * get_const_values() const noexcept
Returns a pointer to the array of values of the matrix.
Definition: dense.hpp:870
gko::make_array_view
array< ValueType > make_array_view(std::shared_ptr< const Executor > exec, size_type size, ValueType *data)
Helper function to create an array view deducing the value type.
Definition: array.hpp:801
gko::matrix::Dense::extract_diagonal
std::unique_ptr< Diagonal< ValueType > > extract_diagonal() const override
Extracts the diagonal entries of the matrix into a vector.
gko::next_precision
typename detail::find_precision_impl< T, step >::type next_precision
Obtains the next move type of T in the singly-linked precision corresponding bfloat16/half.
Definition: math.hpp:466
gko::matrix::Dense::compute_absolute_inplace
void compute_absolute_inplace() override
Compute absolute inplace on each element.
gko::matrix::Dense::column_permute
std::unique_ptr< LinOp > column_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the column permutation of the Permutable object.
gko::previous_precision
typename detail::find_precision_impl< T, -step >::type previous_precision
Obtains the previous move type of T in the singly-linked precision corresponding bfloat16/half.
Definition: math.hpp:473
gko::matrix::Dense::create_const_view_of
static std::unique_ptr< const Dense > create_const_view_of(ptr_param< const Dense > other)
Creates a immutable Dense matrix, where the underlying array is a view of another Dense matrix' array...
Definition: dense.hpp:280
gko::matrix::Dense::at
ValueType & at(size_type idx) noexcept
Returns a single element of the matrix.
Definition: dense.hpp:929
gko::matrix::Dense::conj_transpose
std::unique_ptr< LinOp > conj_transpose() const override
Returns a LinOp representing the conjugate transpose of the Transposable object.
gko::int64
std::int64_t int64
64-bit signed integral type.
Definition: types.hpp:113
gko::matrix::Dense::compute_conj_dot
void compute_conj_dot(ptr_param< const LinOp > b, ptr_param< LinOp > result) const
Computes the column-wise dot product of conj(this matrix) and b.
gko::matrix::Ell
ELL is a matrix format where stride with explicit zeros is used such that all rows have the same numb...
Definition: csr.hpp:31
gko::ConvertibleTo
ConvertibleTo interface is used to mark that the implementer can be converted to the object of Result...
Definition: polymorphic_object.hpp:479
gko::matrix::Dense::create_submatrix
std::unique_ptr< Dense > create_submatrix(const span &rows, const span &columns)
Create a submatrix from the original matrix.
Definition: dense.hpp:1146
gko::int32
std::int32_t int32
32-bit signed integral type.
Definition: types.hpp:107
gko::matrix::Dense::compute_absolute
std::unique_ptr< absolute_type > compute_absolute() const override
Gets the AbsoluteLinOp.
gko::make_dense_view
std::unique_ptr< matrix::Dense< typename detail::pointee< VecPtr >::value_type > > make_dense_view(VecPtr &&vector)
Creates a view of a given Dense vector.
Definition: dense.hpp:1548
gko::Executor
The first step in using the Ginkgo library consists of creating an executor.
Definition: executor.hpp:615
gko::matrix::Hybrid
HYBRID is a matrix format which splits the matrix into ELLPACK and COO format.
Definition: coo.hpp:32
gko::array::get_const_data
const value_type * get_const_data() const noexcept
Returns a constant pointer to the block of memory used to store the elements of the array.
Definition: array.hpp:696
gko::matrix::permute_mode
permute_mode
Specifies how a permutation will be applied to a matrix.
Definition: permutation.hpp:42
gko::matrix::Dense::Dense
Dense(const Dense &)
Copy-constructs a Dense matrix.
gko::matrix::Sellp
SELL-P is a matrix format similar to ELL format.
Definition: csr.hpp:37
gko::make_const_dense_view
std::unique_ptr< const matrix::Dense< typename detail::pointee< VecPtr >::value_type > > make_const_dense_view(VecPtr &&vector)
Creates a view of a given Dense vector.
Definition: dense.hpp:1565
gko::PolymorphicObject::get_executor
std::shared_ptr< const Executor > get_executor() const noexcept
Returns the Executor of the object.
Definition: polymorphic_object.hpp:243
gko::array::get_size
size_type get_size() const noexcept
Returns the number of elements in the array.
Definition: array.hpp:670
gko::LinOp::get_size
const dim< 2 > & get_size() const noexcept
Returns the size of the operator.
Definition: lin_op.hpp:210
gko::local_span
A span that is used exclusively for local numbering.
Definition: range.hpp:138
gko::matrix::Dense::create_submatrix
std::unique_ptr< Dense > create_submatrix(const local_span &rows, const local_span &columns, dim< 2 > size)
Create a submatrix from the original matrix.
Definition: dense.hpp:1161
gko::matrix::Dense::create_real_view
std::unique_ptr< real_type > create_real_view()
Create a real view of the (potentially) complex original matrix.
gko::LinOp::LinOp
LinOp(const LinOp &)=default
Copy-constructs a LinOp.
gko::to_complex
typename detail::to_complex_s< T >::type to_complex
Obtain the type which adds the complex of complex/scalar type or the template parameter of class by a...
Definition: math.hpp:283
gko::matrix::Coo
COO stores a matrix in the coordinate matrix format.
Definition: coo.hpp:50