Ginkgo  Generated from pipelines/2017069469 branch based on develop. Ginkgo version 1.11.0
A numerical linear algebra library targeting many-core architectures
dense.hpp
1 // SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors
2 //
3 // SPDX-License-Identifier: BSD-3-Clause
4 
5 #ifndef GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
6 #define GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
7 
8 
9 #include <initializer_list>
10 #include <type_traits>
11 
12 #include <ginkgo/core/base/array.hpp>
13 #include <ginkgo/core/base/exception_helpers.hpp>
14 #include <ginkgo/core/base/executor.hpp>
15 #include <ginkgo/core/base/lin_op.hpp>
16 #include <ginkgo/core/base/range_accessors.hpp>
17 #include <ginkgo/core/base/types.hpp>
18 #include <ginkgo/core/base/utils.hpp>
19 #include <ginkgo/core/matrix/permutation.hpp>
20 #include <ginkgo/core/matrix/scaled_permutation.hpp>
21 
22 
23 namespace gko {
24 namespace experimental {
25 namespace distributed {
26 
27 
28 template <typename ValueType>
29 class Vector;
30 
31 
32 namespace detail {
33 
34 
35 template <typename ValueType>
36 class VectorCache;
37 
38 
39 } // namespace detail
40 } // namespace distributed
41 } // namespace experimental
42 
43 
44 namespace matrix {
45 
46 
47 template <typename ValueType, typename IndexType>
48 class Coo;
49 
50 template <typename ValueType, typename IndexType>
51 class Csr;
52 
53 template <typename ValueType>
54 class Diagonal;
55 
56 template <typename ValueType, typename IndexType>
57 class Ell;
58 
59 template <typename ValueType, typename IndexType>
60 class Fbcsr;
61 
62 template <typename ValueType, typename IndexType>
63 class Hybrid;
64 
65 template <typename ValueType, typename IndexType>
66 class Sellp;
67 
68 template <typename ValueType, typename IndexType>
69 class SparsityCsr;
70 
71 
87 template <typename ValueType = default_precision>
88 class Dense
89  : public EnableLinOp<Dense<ValueType>>,
90  public ConvertibleTo<Dense<next_precision<ValueType>>>,
91 #if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16
92  public ConvertibleTo<Dense<next_precision<ValueType, 2>>>,
93 #endif
94 #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16
95  public ConvertibleTo<Dense<next_precision<ValueType, 3>>>,
96 #endif
97  public ConvertibleTo<Coo<ValueType, int32>>,
98  public ConvertibleTo<Coo<ValueType, int64>>,
99  public ConvertibleTo<Csr<ValueType, int32>>,
100  public ConvertibleTo<Csr<ValueType, int64>>,
101  public ConvertibleTo<Ell<ValueType, int32>>,
102  public ConvertibleTo<Ell<ValueType, int64>>,
103  public ConvertibleTo<Fbcsr<ValueType, int32>>,
104  public ConvertibleTo<Fbcsr<ValueType, int64>>,
105  public ConvertibleTo<Hybrid<ValueType, int32>>,
106  public ConvertibleTo<Hybrid<ValueType, int64>>,
107  public ConvertibleTo<Sellp<ValueType, int32>>,
108  public ConvertibleTo<Sellp<ValueType, int64>>,
109  public ConvertibleTo<SparsityCsr<ValueType, int32>>,
110  public ConvertibleTo<SparsityCsr<ValueType, int64>>,
111  public DiagonalExtractable<ValueType>,
112  public ReadableFromMatrixData<ValueType, int32>,
113  public ReadableFromMatrixData<ValueType, int64>,
114  public WritableToMatrixData<ValueType, int32>,
115  public WritableToMatrixData<ValueType, int64>,
116  public Transposable,
117  public Permutable<int32>,
118  public Permutable<int64>,
119  public EnableAbsoluteComputation<remove_complex<Dense<ValueType>>>,
120  public ScaledIdentityAddable {
121  friend class EnablePolymorphicObject<Dense, LinOp>;
122  friend class Coo<ValueType, int32>;
123  friend class Coo<ValueType, int64>;
124  friend class Csr<ValueType, int32>;
125  friend class Csr<ValueType, int64>;
126  friend class Diagonal<ValueType>;
127  friend class Ell<ValueType, int32>;
128  friend class Ell<ValueType, int64>;
129  friend class Fbcsr<ValueType, int32>;
130  friend class Fbcsr<ValueType, int64>;
131  friend class Hybrid<ValueType, int32>;
132  friend class Hybrid<ValueType, int64>;
133  friend class Sellp<ValueType, int32>;
134  friend class Sellp<ValueType, int64>;
135  friend class SparsityCsr<ValueType, int32>;
136  friend class SparsityCsr<ValueType, int64>;
137  friend class Dense<to_complex<ValueType>>;
138  friend class experimental::distributed::Vector<ValueType>;
139  friend class experimental::distributed::detail::VectorCache<ValueType>;
140 
141 public:
144  using ConvertibleTo<Dense<next_precision<ValueType>>>::convert_to;
145  using ConvertibleTo<Dense<next_precision<ValueType>>>::move_to;
146  using ConvertibleTo<Coo<ValueType, int32>>::convert_to;
147  using ConvertibleTo<Coo<ValueType, int32>>::move_to;
148  using ConvertibleTo<Coo<ValueType, int64>>::convert_to;
149  using ConvertibleTo<Coo<ValueType, int64>>::move_to;
150  using ConvertibleTo<Csr<ValueType, int32>>::convert_to;
151  using ConvertibleTo<Csr<ValueType, int32>>::move_to;
152  using ConvertibleTo<Csr<ValueType, int64>>::convert_to;
153  using ConvertibleTo<Csr<ValueType, int64>>::move_to;
154  using ConvertibleTo<Ell<ValueType, int32>>::convert_to;
155  using ConvertibleTo<Ell<ValueType, int32>>::move_to;
156  using ConvertibleTo<Ell<ValueType, int64>>::convert_to;
157  using ConvertibleTo<Ell<ValueType, int64>>::move_to;
158  using ConvertibleTo<Fbcsr<ValueType, int32>>::convert_to;
159  using ConvertibleTo<Fbcsr<ValueType, int32>>::move_to;
160  using ConvertibleTo<Fbcsr<ValueType, int64>>::convert_to;
161  using ConvertibleTo<Fbcsr<ValueType, int64>>::move_to;
162  using ConvertibleTo<Hybrid<ValueType, int32>>::convert_to;
163  using ConvertibleTo<Hybrid<ValueType, int32>>::move_to;
164  using ConvertibleTo<Hybrid<ValueType, int64>>::convert_to;
165  using ConvertibleTo<Hybrid<ValueType, int64>>::move_to;
166  using ConvertibleTo<Sellp<ValueType, int32>>::convert_to;
167  using ConvertibleTo<Sellp<ValueType, int32>>::move_to;
168  using ConvertibleTo<Sellp<ValueType, int64>>::convert_to;
169  using ConvertibleTo<Sellp<ValueType, int64>>::move_to;
170  using ConvertibleTo<SparsityCsr<ValueType, int32>>::convert_to;
171  using ConvertibleTo<SparsityCsr<ValueType, int32>>::move_to;
172  using ConvertibleTo<SparsityCsr<ValueType, int64>>::convert_to;
173  using ConvertibleTo<SparsityCsr<ValueType, int64>>::move_to;
176 
177  using value_type = ValueType;
178  using index_type = int64;
179  using transposed_type = Dense<ValueType>;
180  using mat_data = matrix_data<ValueType, int64>;
181  using mat_data32 = matrix_data<ValueType, int32>;
182  using device_mat_data = device_matrix_data<ValueType, int64>;
183  using device_mat_data32 = device_matrix_data<ValueType, int32>;
184  using absolute_type = remove_complex<Dense>;
185  using real_type = absolute_type;
186  using complex_type = to_complex<Dense>;
187 
188  using row_major_range = gko::range<gko::accessor::row_major<ValueType, 2>>;
189 
196  static std::unique_ptr<Dense> create_with_config_of(
198  {
199  // De-referencing `other` before calling the functions (instead of
200  // using operator `->`) is currently required to be compatible with
201  // CUDA 10.1.
202  // Otherwise, it results in a compile error.
203  return (*other).create_with_same_config();
204  }
205 
217  static std::unique_ptr<Dense> create_with_type_of(
218  ptr_param<const Dense> other, std::shared_ptr<const Executor> exec,
219  const dim<2>& size = dim<2>{})
220  {
221  // See create_with_config_of()
222  return (*other).create_with_type_of_impl(exec, size, size[1]);
223  }
224 
233  static std::unique_ptr<Dense> create_with_type_of(
234  ptr_param<const Dense> other, std::shared_ptr<const Executor> exec,
235  const dim<2>& size, size_type stride)
236  {
237  // See create_with_config_of()
238  return (*other).create_with_type_of_impl(exec, size, stride);
239  }
240 
251  static std::unique_ptr<Dense> create_with_type_of(
252  ptr_param<const Dense> other, std::shared_ptr<const Executor> exec,
253  const dim<2>& size, const dim<2>& local_size, size_type stride)
254  {
255  // See create_with_config_of()
256  return (*other).create_with_type_of_impl(exec, size, stride);
257  }
258 
267  static std::unique_ptr<Dense> create_view_of(ptr_param<Dense> other)
268  {
269  return other->create_view_of_impl();
270  }
271 
279  static std::unique_ptr<const Dense> create_const_view_of(
281  {
282  return other->create_const_view_of_impl();
283  }
284 
285  friend class Dense<previous_precision<ValueType>>;
286 
287  void convert_to(Dense<next_precision<ValueType>>* result) const override;
288 
289  void move_to(Dense<next_precision<ValueType>>* result) override;
290 
291 #if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16
292  friend class Dense<previous_precision<ValueType, 2>>;
295 
296  void convert_to(Dense<next_precision<ValueType, 2>>* result) const override;
297 
298  void move_to(Dense<next_precision<ValueType, 2>>* result) override;
299 #endif
300 
301 #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16
302  friend class Dense<previous_precision<ValueType, 3>>;
305 
306  void convert_to(Dense<next_precision<ValueType, 3>>* result) const override;
307 
308  void move_to(Dense<next_precision<ValueType, 3>>* result) override;
309 #endif
310 
311  void convert_to(Coo<ValueType, int32>* result) const override;
312 
313  void move_to(Coo<ValueType, int32>* result) override;
314 
315  void convert_to(Coo<ValueType, int64>* result) const override;
316 
317  void move_to(Coo<ValueType, int64>* result) override;
318 
319  void convert_to(Csr<ValueType, int32>* result) const override;
320 
321  void move_to(Csr<ValueType, int32>* result) override;
322 
323  void convert_to(Csr<ValueType, int64>* result) const override;
324 
325  void move_to(Csr<ValueType, int64>* result) override;
326 
327  void convert_to(Ell<ValueType, int32>* result) const override;
328 
329  void move_to(Ell<ValueType, int32>* result) override;
330 
331  void convert_to(Ell<ValueType, int64>* result) const override;
332 
333  void move_to(Ell<ValueType, int64>* result) override;
334 
335  void convert_to(Fbcsr<ValueType, int32>* result) const override;
336 
337  void move_to(Fbcsr<ValueType, int32>* result) override;
338 
339  void convert_to(Fbcsr<ValueType, int64>* result) const override;
340 
341  void move_to(Fbcsr<ValueType, int64>* result) override;
342 
343  void convert_to(Hybrid<ValueType, int32>* result) const override;
344 
345  void move_to(Hybrid<ValueType, int32>* result) override;
346 
347  void convert_to(Hybrid<ValueType, int64>* result) const override;
348 
349  void move_to(Hybrid<ValueType, int64>* result) override;
350 
351  void convert_to(Sellp<ValueType, int32>* result) const override;
352 
353  void move_to(Sellp<ValueType, int32>* result) override;
354 
355  void convert_to(Sellp<ValueType, int64>* result) const override;
356 
357  void move_to(Sellp<ValueType, int64>* result) override;
358 
359  void convert_to(SparsityCsr<ValueType, int32>* result) const override;
360 
361  void move_to(SparsityCsr<ValueType, int32>* result) override;
362 
363  void convert_to(SparsityCsr<ValueType, int64>* result) const override;
364 
365  void move_to(SparsityCsr<ValueType, int64>* result) override;
366 
367  void read(const mat_data& data) override;
368 
369  void read(const mat_data32& data) override;
370 
371  void read(const device_mat_data& data) override;
372 
373  void read(const device_mat_data32& data) override;
374 
375  void read(device_mat_data&& data) override;
376 
377  void read(device_mat_data32&& data) override;
378 
379  void write(mat_data& data) const override;
380 
381  void write(mat_data32& data) const override;
382 
383  std::unique_ptr<LinOp> transpose() const override;
384 
385  std::unique_ptr<LinOp> conj_transpose() const override;
386 
393  void transpose(ptr_param<Dense> output) const;
394 
401  void conj_transpose(ptr_param<Dense> output) const;
402 
408  void fill(const ValueType value);
409 
424  std::unique_ptr<Dense> permute(
425  ptr_param<const Permutation<int32>> permutation,
427 
431  std::unique_ptr<Dense> permute(
432  ptr_param<const Permutation<int64>> permutation,
434 
440  void permute(ptr_param<const Permutation<int32>> permutation,
441  ptr_param<Dense> output, permute_mode mode) const;
442 
447  void permute(ptr_param<const Permutation<int64>> permutation,
448  ptr_param<Dense> output, permute_mode mode) const;
449 
463  std::unique_ptr<Dense> permute(
464  ptr_param<const Permutation<int32>> row_permutation,
465  ptr_param<const Permutation<int32>> column_permutation,
466  bool invert = false) const;
467 
472  std::unique_ptr<Dense> permute(
473  ptr_param<const Permutation<int64>> row_permutation,
474  ptr_param<const Permutation<int64>> column_permutation,
475  bool invert = false) const;
476 
483  void permute(ptr_param<const Permutation<int32>> row_permutation,
484  ptr_param<const Permutation<int32>> column_permutation,
485  ptr_param<Dense> output, bool invert = false) const;
486 
491  void permute(ptr_param<const Permutation<int64>> row_permutation,
492  ptr_param<const Permutation<int64>> column_permutation,
493  ptr_param<Dense> output, bool invert = false) const;
494 
504  std::unique_ptr<Dense> scale_permute(
507 
512  std::unique_ptr<Dense> scale_permute(
515 
522  void scale_permute(
524  ptr_param<Dense> output, permute_mode mode) const;
525 
530  void scale_permute(
532  ptr_param<Dense> output, permute_mode mode) const;
533 
546  std::unique_ptr<Dense> scale_permute(
547  ptr_param<const ScaledPermutation<value_type, int32>> row_permutation,
549  column_permutation,
550  bool invert = false) const;
551 
556  std::unique_ptr<Dense> scale_permute(
557  ptr_param<const ScaledPermutation<value_type, int64>> row_permutation,
559  column_permutation,
560  bool invert = false) const;
561 
568  void scale_permute(
569  ptr_param<const ScaledPermutation<value_type, int32>> row_permutation,
571  column_permutation,
572  ptr_param<Dense> output, bool invert = false) const;
573 
579  void scale_permute(
580  ptr_param<const ScaledPermutation<value_type, int64>> row_permutation,
582  column_permutation,
583  ptr_param<Dense> output, bool invert = false) const;
584 
585  std::unique_ptr<LinOp> permute(
586  const array<int32>* permutation_indices) const override;
587 
588  std::unique_ptr<LinOp> permute(
589  const array<int64>* permutation_indices) const override;
590 
600  void permute(const array<int32>* permutation_indices,
601  ptr_param<Dense> output) const;
602 
606  void permute(const array<int64>* permutation_indices,
607  ptr_param<Dense> output) const;
608 
609  std::unique_ptr<LinOp> inverse_permute(
610  const array<int32>* permutation_indices) const override;
611 
612  std::unique_ptr<LinOp> inverse_permute(
613  const array<int64>* permutation_indices) const override;
614 
625  void inverse_permute(const array<int32>* permutation_indices,
626  ptr_param<Dense> output) const;
627 
631  void inverse_permute(const array<int64>* permutation_indices,
632  ptr_param<Dense> output) const;
633 
634  std::unique_ptr<LinOp> row_permute(
635  const array<int32>* permutation_indices) const override;
636 
637  std::unique_ptr<LinOp> row_permute(
638  const array<int64>* permutation_indices) const override;
639 
649  void row_permute(const array<int32>* permutation_indices,
650  ptr_param<Dense> output) const;
651 
655  void row_permute(const array<int64>* permutation_indices,
656  ptr_param<Dense> output) const;
657 
668  std::unique_ptr<Dense> row_gather(const array<int32>* gather_indices) const;
669 
673  std::unique_ptr<Dense> row_gather(const array<int64>* gather_indices) const;
674 
687  void row_gather(const array<int32>* gather_indices,
688  ptr_param<LinOp> row_collection) const;
689 
693  void row_gather(const array<int64>* gather_indices,
694  ptr_param<LinOp> row_collection) const;
695 
710  const array<int32>* gather_indices,
712  ptr_param<LinOp> row_collection) const;
713 
719  const array<int64>* gather_indices,
721  ptr_param<LinOp> row_collection) const;
722 
723  std::unique_ptr<LinOp> column_permute(
724  const array<int32>* permutation_indices) const override;
725 
726  std::unique_ptr<LinOp> column_permute(
727  const array<int64>* permutation_indices) const override;
728 
738  void column_permute(const array<int32>* permutation_indices,
739  ptr_param<Dense> output) const;
740 
744  void column_permute(const array<int64>* permutation_indices,
745  ptr_param<Dense> output) const;
746 
747  std::unique_ptr<LinOp> inverse_row_permute(
748  const array<int32>* permutation_indices) const override;
749 
750  std::unique_ptr<LinOp> inverse_row_permute(
751  const array<int64>* permutation_indices) const override;
752 
762  void inverse_row_permute(const array<int32>* permutation_indices,
763  ptr_param<Dense> output) const;
764 
768  void inverse_row_permute(const array<int64>* permutation_indices,
769  ptr_param<Dense> output) const;
770 
771  std::unique_ptr<LinOp> inverse_column_permute(
772  const array<int32>* permutation_indices) const override;
773 
774  std::unique_ptr<LinOp> inverse_column_permute(
775  const array<int64>* permutation_indices) const override;
776 
786  void inverse_column_permute(const array<int32>* permutation_indices,
787  ptr_param<Dense> output) const;
788 
792  void inverse_column_permute(const array<int64>* permutation_indices,
793  ptr_param<Dense> output) const;
794 
795  std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
796 
804  void extract_diagonal(ptr_param<Diagonal<ValueType>> output) const;
805 
806  std::unique_ptr<absolute_type> compute_absolute() const override;
807 
815  void compute_absolute(ptr_param<absolute_type> output) const;
816 
817  void compute_absolute_inplace() override;
818 
823  std::unique_ptr<complex_type> make_complex() const;
824 
830  void make_complex(ptr_param<complex_type> result) const;
831 
836  std::unique_ptr<real_type> get_real() const;
837 
841  void get_real(ptr_param<real_type> result) const;
842 
847  std::unique_ptr<real_type> get_imag() const;
848 
853  void get_imag(ptr_param<real_type> result) const;
854 
860  value_type* get_values() noexcept { return values_.get_data(); }
861 
869  const value_type* get_const_values() const noexcept
870  {
871  return values_.get_const_data();
872  }
873 
879  size_type get_stride() const noexcept { return stride_; }
880 
887  {
888  return values_.get_size();
889  }
890 
901  value_type& at(size_type row, size_type col) noexcept
902  {
903  return values_.get_data()[linearize_index(row, col)];
904  }
905 
909  value_type at(size_type row, size_type col) const noexcept
910  {
911  return values_.get_const_data()[linearize_index(row, col)];
912  }
913 
928  ValueType& at(size_type idx) noexcept
929  {
930  return values_.get_data()[linearize_index(idx)];
931  }
932 
936  ValueType at(size_type idx) const noexcept
937  {
938  return values_.get_const_data()[linearize_index(idx)];
939  }
940 
950  void scale(ptr_param<const LinOp> alpha);
951 
961  void inv_scale(ptr_param<const LinOp> alpha);
962 
974 
986 
996 
1009  array<char>& tmp) const;
1010 
1020  ptr_param<LinOp> result) const;
1021 
1034  array<char>& tmp) const;
1035 
1043  void compute_norm2(ptr_param<LinOp> result) const;
1044 
1055  void compute_norm2(ptr_param<LinOp> result, array<char>& tmp) const;
1056 
1064  void compute_norm1(ptr_param<LinOp> result) const;
1065 
1076  void compute_norm1(ptr_param<LinOp> result, array<char>& tmp) const;
1077 
1086  void compute_squared_norm2(ptr_param<LinOp> result) const;
1087 
1099  void compute_squared_norm2(ptr_param<LinOp> result, array<char>& tmp) const;
1100 
1108  void compute_mean(ptr_param<LinOp> result) const;
1109 
1120  void compute_mean(ptr_param<LinOp> result, array<char>& tmp) const;
1121 
1132  std::unique_ptr<Dense> create_submatrix(const span& rows,
1133  const span& columns,
1134  const size_type stride)
1135  {
1136  return this->create_submatrix_impl(rows, columns, stride);
1137  }
1138 
1145  std::unique_ptr<Dense> create_submatrix(const span& rows,
1146  const span& columns)
1147  {
1148  return create_submatrix(rows, columns, this->get_stride());
1149  }
1150 
1151 
1160  std::unique_ptr<Dense> create_submatrix(const local_span& rows,
1161  const local_span& columns,
1162  dim<2> size)
1163  {
1164  dim<2> deduced_size{rows.length(), columns.length()};
1165  GKO_ASSERT_EQUAL_DIMENSIONS(deduced_size, size);
1166  return create_submatrix(rows, columns, this->get_stride());
1167  }
1168 
1176  std::unique_ptr<real_type> create_real_view();
1177 
1181  std::unique_ptr<const real_type> create_real_view() const;
1182 
1195  static std::unique_ptr<Dense> create(std::shared_ptr<const Executor> exec,
1196  const dim<2>& size = {},
1197  size_type stride = 0);
1198 
1215  static std::unique_ptr<Dense> create(std::shared_ptr<const Executor> exec,
1216  const dim<2>& size,
1217  array<value_type> values,
1218  size_type stride);
1219 
1224  template <typename InputValueType>
1225  GKO_DEPRECATED(
1226  "explicitly construct the gko::array argument instead of passing an"
1227  "initializer list")
1228  static std::unique_ptr<Dense> create(
1229  std::shared_ptr<const Executor> exec, const dim<2>& size,
1230  std::initializer_list<InputValueType> values, size_type stride)
1231  {
1232  return create(exec, size, array<value_type>{exec, std::move(values)},
1233  stride);
1234  }
1235 
1247  static std::unique_ptr<const Dense> create_const(
1248  std::shared_ptr<const Executor> exec, const dim<2>& size,
1249  gko::detail::const_array_view<ValueType>&& values, size_type stride);
1250 
1256  Dense& operator=(const Dense&);
1257 
1263  Dense& operator=(Dense&&);
1264 
1269  Dense(const Dense&);
1270 
1275  Dense(Dense&&);
1276 
1277 protected:
1278  Dense(std::shared_ptr<const Executor> exec, const dim<2>& size = {},
1279  size_type stride = 0);
1280 
1281  Dense(std::shared_ptr<const Executor> exec, const dim<2>& size,
1282  array<value_type> values, size_type stride);
1283 
1290  virtual std::unique_ptr<Dense> create_with_same_config() const
1291  {
1292  return Dense::create(this->get_executor(), this->get_size(),
1293  this->get_stride());
1294  }
1295 
1303  virtual std::unique_ptr<Dense> create_with_type_of_impl(
1304  std::shared_ptr<const Executor> exec, const dim<2>& size,
1305  size_type stride) const
1306  {
1307  return Dense::create(exec, size, stride);
1308  }
1309 
1316  virtual std::unique_ptr<Dense> create_view_of_impl()
1317  {
1318  auto exec = this->get_executor();
1319  return Dense::create(
1320  exec, this->get_size(),
1322  this->get_values()),
1323  this->get_stride());
1324  }
1325 
1332  virtual std::unique_ptr<const Dense> create_const_view_of_impl() const
1333  {
1334  auto exec = this->get_executor();
1335  return Dense::create_const(
1336  exec, this->get_size(),
1338  this->get_const_values()),
1339  this->get_stride());
1340  }
1341 
1342  template <typename IndexType>
1343  void convert_impl(Coo<ValueType, IndexType>* result) const;
1344 
1345  template <typename IndexType>
1346  void convert_impl(Csr<ValueType, IndexType>* result) const;
1347 
1348  template <typename IndexType>
1349  void convert_impl(Ell<ValueType, IndexType>* result) const;
1350 
1351  template <typename IndexType>
1352  void convert_impl(Fbcsr<ValueType, IndexType>* result) const;
1353 
1354  template <typename IndexType>
1355  void convert_impl(Hybrid<ValueType, IndexType>* result) const;
1356 
1357  template <typename IndexType>
1358  void convert_impl(Sellp<ValueType, IndexType>* result) const;
1359 
1360  template <typename IndexType>
1361  void convert_impl(SparsityCsr<ValueType, IndexType>* result) const;
1362 
1369  virtual void scale_impl(const LinOp* alpha);
1370 
1377  virtual void inv_scale_impl(const LinOp* alpha);
1378 
1385  virtual void add_scaled_impl(const LinOp* alpha, const LinOp* b);
1386 
1393  virtual void sub_scaled_impl(const LinOp* alpha, const LinOp* b);
1394 
1401  virtual void compute_dot_impl(const LinOp* b, LinOp* result) const;
1402 
1409  virtual void compute_conj_dot_impl(const LinOp* b, LinOp* result) const;
1410 
1417  virtual void compute_norm2_impl(LinOp* result) const;
1418 
1425  virtual void compute_norm1_impl(LinOp* result) const;
1426 
1433  virtual void compute_squared_norm2_impl(LinOp* result) const;
1434 
1438  virtual void compute_mean_impl(LinOp* result) const;
1439 
1448  void resize(gko::dim<2> new_size);
1449 
1457  virtual std::unique_ptr<Dense> create_submatrix_impl(
1458  const span& rows, const span& columns, const size_type stride);
1459 
1460  void apply_impl(const LinOp* b, LinOp* x) const override;
1461 
1462  void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta,
1463  LinOp* x) const override;
1464 
1465  size_type linearize_index(size_type row, size_type col) const noexcept
1466  {
1467  return row * stride_ + col;
1468  }
1469 
1470  size_type linearize_index(size_type idx) const noexcept
1471  {
1472  return linearize_index(idx / this->get_size()[1],
1473  idx % this->get_size()[1]);
1474  }
1475 
1476  template <typename IndexType>
1477  void permute_impl(const Permutation<IndexType>* permutation,
1478  permute_mode mode, Dense* output) const;
1479 
1480  template <typename IndexType>
1481  void permute_impl(const Permutation<IndexType>* row_permutation,
1482  const Permutation<IndexType>* col_permutation,
1483  bool invert, Dense* output) const;
1484 
1485  template <typename IndexType>
1486  void scale_permute_impl(
1487  const ScaledPermutation<ValueType, IndexType>* permutation,
1488  permute_mode mode, Dense* output) const;
1489 
1490  template <typename IndexType>
1491  void scale_permute_impl(
1492  const ScaledPermutation<ValueType, IndexType>* row_permutation,
1493  const ScaledPermutation<ValueType, IndexType>* column_permutation,
1494  bool invert, Dense* output) const;
1495 
1496  template <typename OutputType, typename IndexType>
1497  void row_gather_impl(const array<IndexType>* row_idxs,
1498  Dense<OutputType>* row_collection) const;
1499 
1500  template <typename OutputType, typename IndexType>
1501  void row_gather_impl(const Dense<ValueType>* alpha,
1502  const array<IndexType>* row_idxs,
1503  const Dense<ValueType>* beta,
1504  Dense<OutputType>* row_collection) const;
1505 
1506 private:
1507  size_type stride_;
1508  array<value_type> values_;
1509 
1510  void add_scaled_identity_impl(const LinOp* a, const LinOp* b) override;
1511 };
1512 
1513 
1514 } // namespace matrix
1515 
1516 
1517 namespace detail {
1518 
1519 
1520 template <typename ValueType>
1521 struct temporary_clone_helper<matrix::Dense<ValueType>> {
1522  static std::unique_ptr<matrix::Dense<ValueType>> create(
1523  std::shared_ptr<const Executor> exec, matrix::Dense<ValueType>* ptr,
1524  bool copy_data)
1525  {
1526  if (copy_data) {
1527  return gko::clone(std::move(exec), ptr);
1528  } else {
1529  return matrix::Dense<ValueType>::create(exec, ptr->get_size());
1530  }
1531  }
1532 };
1533 
1534 
1535 } // namespace detail
1536 
1537 
1545 template <typename VecPtr>
1546 std::unique_ptr<matrix::Dense<typename detail::pointee<VecPtr>::value_type>>
1547 make_dense_view(VecPtr&& vector)
1548 {
1549  using value_type = typename detail::pointee<VecPtr>::value_type;
1551 }
1552 
1553 
1561 template <typename VecPtr>
1562 std::unique_ptr<
1563  const matrix::Dense<typename detail::pointee<VecPtr>::value_type>>
1564 make_const_dense_view(VecPtr&& vector)
1565 {
1566  using value_type = typename detail::pointee<VecPtr>::value_type;
1568 }
1569 
1570 
1591 template <typename Matrix, typename... TArgs>
1592 std::unique_ptr<Matrix> initialize(
1593  size_type stride, std::initializer_list<typename Matrix::value_type> vals,
1594  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1595 {
1597  size_type num_rows = vals.size();
1598  auto tmp = dense::create(exec->get_master(), dim<2>{num_rows, 1}, stride);
1599  size_type idx = 0;
1600  for (const auto& elem : vals) {
1601  tmp->at(idx) = elem;
1602  ++idx;
1603  }
1604  auto mtx = Matrix::create(exec, std::forward<TArgs>(create_args)...);
1605  tmp->move_to(mtx);
1606  return mtx;
1607 }
1608 
1629 template <typename Matrix, typename... TArgs>
1630 std::unique_ptr<Matrix> initialize(
1631  std::initializer_list<typename Matrix::value_type> vals,
1632  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1633 {
1634  return initialize<Matrix>(1, vals, std::move(exec),
1635  std::forward<TArgs>(create_args)...);
1636 }
1637 
1638 
1659 template <typename Matrix, typename... TArgs>
1660 std::unique_ptr<Matrix> initialize(
1661  size_type stride,
1662  std::initializer_list<std::initializer_list<typename Matrix::value_type>>
1663  vals,
1664  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1665 {
1667  size_type num_rows = vals.size();
1668  size_type num_cols = num_rows > 0 ? begin(vals)->size() : 1;
1669  auto tmp =
1670  dense::create(exec->get_master(), dim<2>{num_rows, num_cols}, stride);
1671  size_type ridx = 0;
1672  for (const auto& row : vals) {
1673  size_type cidx = 0;
1674  for (const auto& elem : row) {
1675  tmp->at(ridx, cidx) = elem;
1676  ++cidx;
1677  }
1678  ++ridx;
1679  }
1680  auto mtx = Matrix::create(exec, std::forward<TArgs>(create_args)...);
1681  tmp->move_to(mtx);
1682  return mtx;
1683 }
1684 
1685 
1707 template <typename Matrix, typename... TArgs>
1708 std::unique_ptr<Matrix> initialize(
1709  std::initializer_list<std::initializer_list<typename Matrix::value_type>>
1710  vals,
1711  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1712 {
1713  return initialize<Matrix>(vals.size() > 0 ? begin(vals)->size() : 0, vals,
1714  std::move(exec),
1715  std::forward<TArgs>(create_args)...);
1716 }
1717 
1718 
1719 } // namespace gko
1720 
1721 
1722 #endif // GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
gko::matrix::Dense::permute
std::unique_ptr< Dense > permute(ptr_param< const Permutation< int32 >> permutation, permute_mode mode=permute_mode::symmetric) const
Creates a permuted copy of this matrix with the given permutation .
gko::matrix::Dense::inverse_column_permute
std::unique_ptr< LinOp > inverse_column_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the row permutation of the inverse permuted object.
gko::matrix::Dense::add_scaled
void add_scaled(ptr_param< const LinOp > alpha, ptr_param< const LinOp > b)
Adds b scaled by alpha to the matrix (aka: BLAS axpy).
gko::matrix::Fbcsr
Fixed-block compressed sparse row storage matrix format.
Definition: csr.hpp:46
gko::EnablePolymorphicAssignment< ConcreteLinOp >::move_to
void move_to(result_type *result) override
Definition: polymorphic_object.hpp:751
gko::EnablePolymorphicAssignment< ConcreteLinOp >::convert_to
void convert_to(result_type *result) const override
Definition: polymorphic_object.hpp:749
gko::matrix::Csr
CSR is a matrix format which stores only the nonzero coefficients by compressing each row of the matr...
Definition: matrix.hpp:30
gko::matrix::permute_mode::rows
The rows will be permuted.
gko::matrix::Dense::transpose
std::unique_ptr< LinOp > transpose() const override
Returns a LinOp representing the transpose of the Transposable object.
gko::matrix::permute_mode::columns
The columns will be permuted.
gko::ReadableFromMatrixData< ValueType, int32 >::read
virtual void read(const matrix_data< ValueType, int32 > &data)=0
Reads a matrix from a matrix_data structure.
gko::matrix::Dense
Dense is a matrix format which explicitly stores all values of the matrix.
Definition: dense_cache.hpp:28
gko::matrix::SparsityCsr
SparsityCsr is a matrix format which stores only the sparsity pattern of a sparse matrix by compressi...
Definition: csr.hpp:40
gko::matrix::Dense::at
ValueType at(size_type idx) const noexcept
Returns a single element of the matrix.
Definition: dense.hpp:936
gko::matrix::Dense::row_gather
std::unique_ptr< Dense > row_gather(const array< int32 > *gather_indices) const
Create a Dense matrix consisting of the given rows from this matrix.
gko::matrix::Dense::make_complex
std::unique_ptr< complex_type > make_complex() const
Creates a complex copy of the original matrix.
gko::matrix::Dense::create
static std::unique_ptr< Dense > create(std::shared_ptr< const Executor > exec, const dim< 2 > &size={}, size_type stride=0)
Creates an uninitialized Dense matrix of the specified size.
gko::matrix::Dense::create_with_type_of
static std::unique_ptr< Dense > create_with_type_of(ptr_param< const Dense > other, std::shared_ptr< const Executor > exec, const dim< 2 > &size, size_type stride)
Definition: dense.hpp:233
gko::matrix::Dense::scale
void scale(ptr_param< const LinOp > alpha)
Scales the matrix with a scalar (aka: BLAS scal).
gko::matrix::ScaledPermutation
ScaledPermutation is a matrix combining a permutation with scaling factors.
Definition: scaled_permutation.hpp:36
gko::size_type
std::size_t size_type
Integral type used for allocation quantities.
Definition: types.hpp:90
gko::matrix::Dense::get_real
std::unique_ptr< real_type > get_real() const
Creates a new real matrix and extracts the real part of the original matrix into that.
gko::matrix::Dense::inv_scale
void inv_scale(ptr_param< const LinOp > alpha)
Scales the matrix with the inverse of a scalar.
gko::matrix::Permutation
Permutation is a matrix format that represents a permutation matrix, i.e.
Definition: csr.hpp:52
gko::matrix::Dense::fill
void fill(const ValueType value)
Fill the dense matrix with a given value.
gko::make_const_array_view
detail::const_array_view< ValueType > make_const_array_view(std::shared_ptr< const Executor > exec, size_type size, const ValueType *data)
Helper function to create a const array view deducing the value type.
Definition: array.hpp:820
gko::matrix::Dense::compute_dot
void compute_dot(ptr_param< const LinOp > b, ptr_param< LinOp > result) const
Computes the column-wise dot product of this matrix and b.
gko::matrix::Dense::create_submatrix
std::unique_ptr< Dense > create_submatrix(const span &rows, const span &columns, const size_type stride)
Create a submatrix from the original matrix.
Definition: dense.hpp:1132
gko::matrix::Dense::create_with_config_of
static std::unique_ptr< Dense > create_with_config_of(ptr_param< const Dense > other)
Creates a Dense matrix with the same size and stride as another Dense matrix.
Definition: dense.hpp:196
gko::initialize
std::unique_ptr< Matrix > initialize(size_type stride, std::initializer_list< typename Matrix::value_type > vals, std::shared_ptr< const Executor > exec, TArgs &&... create_args)
Creates and initializes a column-vector.
Definition: dense.hpp:1592
gko::matrix::Dense::at
value_type & at(size_type row, size_type col) noexcept
Returns a single element of the matrix.
Definition: dense.hpp:901
gko::clone
detail::cloned_type< Pointer > clone(const Pointer &p)
Creates a unique clone of the object pointed to by p.
Definition: utils_helper.hpp:173
gko::matrix::Dense::get_stride
size_type get_stride() const noexcept
Returns the stride of the matrix.
Definition: dense.hpp:879
gko::matrix::Dense::create_view_of
static std::unique_ptr< Dense > create_view_of(ptr_param< Dense > other)
Creates a Dense matrix, where the underlying array is a view of another Dense matrix' array.
Definition: dense.hpp:267
gko::range
A range is a multidimensional view of the memory.
Definition: range.hpp:304
gko
The Ginkgo namespace.
Definition: abstract_factory.hpp:20
gko::matrix::Dense::get_num_stored_elements
size_type get_num_stored_elements() const noexcept
Returns the number of elements explicitly stored in the matrix.
Definition: dense.hpp:886
gko::matrix::Dense::sub_scaled
void sub_scaled(ptr_param< const LinOp > alpha, ptr_param< const LinOp > b)
Subtracts b scaled by alpha from the matrix (aka: BLAS axpy).
gko::matrix::Dense::get_values
value_type * get_values() noexcept
Returns a pointer to the array of values of the matrix.
Definition: dense.hpp:860
gko::array
An array is a container which encapsulates fixed-sized arrays, stored on the Executor tied to the arr...
Definition: array.hpp:26
gko::matrix::Dense::compute_norm2
void compute_norm2(ptr_param< LinOp > result) const
Computes the column-wise Euclidean (L^2) norm of this matrix.
gko::matrix::Dense::row_permute
std::unique_ptr< LinOp > row_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the row permutation of the Permutable object.
gko::matrix::Dense::create_with_type_of
static std::unique_ptr< Dense > create_with_type_of(ptr_param< const Dense > other, std::shared_ptr< const Executor > exec, const dim< 2 > &size=dim< 2 >{})
Creates a Dense matrix with the same type as another Dense matrix but on a different executor and wit...
Definition: dense.hpp:217
gko::span
A span is a lightweight structure used to create sub-ranges from other ranges.
Definition: range.hpp:46
gko::dim< 2 >
gko::matrix::Diagonal
This class is a utility which efficiently implements the diagonal matrix (a linear operator which sca...
Definition: lin_op.hpp:31
gko::matrix::Dense::create_const
static std::unique_ptr< const Dense > create_const(std::shared_ptr< const Executor > exec, const dim< 2 > &size, gko::detail::const_array_view< ValueType > &&values, size_type stride)
Creates a constant (immutable) Dense matrix from a constant array.
gko::ptr_param
This class is used for function parameters in the place of raw pointers.
Definition: utils_helper.hpp:41
gko::matrix::Dense::at
value_type at(size_type row, size_type col) const noexcept
Returns a single element of the matrix.
Definition: dense.hpp:909
gko::array::get_data
value_type * get_data() noexcept
Returns a pointer to the block of memory used to store the elements of the array.
Definition: array.hpp:687
gko::matrix::Dense::compute_mean
void compute_mean(ptr_param< LinOp > result) const
Computes the column-wise arithmetic mean of this matrix.
gko::matrix::Dense::inverse_row_permute
std::unique_ptr< LinOp > inverse_row_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the row permutation of the inverse permuted object.
gko::matrix::permute_mode::symmetric
The rows and columns will be permuted.
gko::matrix::Dense::get_imag
std::unique_ptr< real_type > get_imag() const
Creates a new real matrix and extracts the imaginary part of the original matrix into that.
gko::matrix::Dense::compute_squared_norm2
void compute_squared_norm2(ptr_param< LinOp > result) const
Computes the square of the column-wise Euclidean (L^2) norm of this matrix.
gko::matrix::Dense::operator=
Dense & operator=(const Dense &)
Copy-assigns a Dense matrix.
gko::matrix::Dense::inverse_permute
std::unique_ptr< LinOp > inverse_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the symmetric inverse row and column permutation of the Permutable objec...
gko::matrix::Dense::create_with_type_of
static std::unique_ptr< Dense > create_with_type_of(ptr_param< const Dense > other, std::shared_ptr< const Executor > exec, const dim< 2 > &size, const dim< 2 > &local_size, size_type stride)
Definition: dense.hpp:251
gko::stop::mode
mode
The mode for the residual norm criterion.
Definition: residual_norm.hpp:38
gko::matrix::Dense::scale_permute
std::unique_ptr< Dense > scale_permute(ptr_param< const ScaledPermutation< value_type, int32 >> permutation, permute_mode mode=permute_mode::symmetric) const
Creates a scaled and permuted copy of this matrix.
gko::matrix::Dense::compute_norm1
void compute_norm1(ptr_param< LinOp > result) const
Computes the column-wise (L^1) norm of this matrix.
gko::matrix::Dense::get_const_values
const value_type * get_const_values() const noexcept
Returns a pointer to the array of values of the matrix.
Definition: dense.hpp:869
gko::make_array_view
array< ValueType > make_array_view(std::shared_ptr< const Executor > exec, size_type size, ValueType *data)
Helper function to create an array view deducing the value type.
Definition: array.hpp:801
gko::matrix::Dense::extract_diagonal
std::unique_ptr< Diagonal< ValueType > > extract_diagonal() const override
Extracts the diagonal entries of the matrix into a vector.
gko::next_precision
typename detail::find_precision_impl< T, step >::type next_precision
Obtains the next move type of T in the singly-linked precision corresponding bfloat16/half.
Definition: math.hpp:466
gko::matrix::Dense::compute_absolute_inplace
void compute_absolute_inplace() override
Compute absolute inplace on each element.
gko::matrix::Dense::column_permute
std::unique_ptr< LinOp > column_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the column permutation of the Permutable object.
gko::previous_precision
typename detail::find_precision_impl< T, -step >::type previous_precision
Obtains the previous move type of T in the singly-linked precision corresponding bfloat16/half.
Definition: math.hpp:473
gko::matrix::Dense::create_const_view_of
static std::unique_ptr< const Dense > create_const_view_of(ptr_param< const Dense > other)
Creates a immutable Dense matrix, where the underlying array is a view of another Dense matrix' array...
Definition: dense.hpp:279
gko::matrix::Dense::at
ValueType & at(size_type idx) noexcept
Returns a single element of the matrix.
Definition: dense.hpp:928
gko::matrix::Dense::conj_transpose
std::unique_ptr< LinOp > conj_transpose() const override
Returns a LinOp representing the conjugate transpose of the Transposable object.
gko::int64
std::int64_t int64
64-bit signed integral type.
Definition: types.hpp:113
gko::matrix::Dense::compute_conj_dot
void compute_conj_dot(ptr_param< const LinOp > b, ptr_param< LinOp > result) const
Computes the column-wise dot product of conj(this matrix) and b.
gko::matrix::Ell
ELL is a matrix format where stride with explicit zeros is used such that all rows have the same numb...
Definition: csr.hpp:31
gko::ConvertibleTo
ConvertibleTo interface is used to mark that the implementer can be converted to the object of Result...
Definition: polymorphic_object.hpp:479
gko::matrix::Dense::create_submatrix
std::unique_ptr< Dense > create_submatrix(const span &rows, const span &columns)
Create a submatrix from the original matrix.
Definition: dense.hpp:1145
gko::int32
std::int32_t int32
32-bit signed integral type.
Definition: types.hpp:107
gko::matrix::Dense::compute_absolute
std::unique_ptr< absolute_type > compute_absolute() const override
Gets the AbsoluteLinOp.
gko::make_dense_view
std::unique_ptr< matrix::Dense< typename detail::pointee< VecPtr >::value_type > > make_dense_view(VecPtr &&vector)
Creates a view of a given Dense vector.
Definition: dense.hpp:1547
gko::Executor
The first step in using the Ginkgo library consists of creating an executor.
Definition: executor.hpp:615
gko::matrix::Hybrid
HYBRID is a matrix format which splits the matrix into ELLPACK and COO format.
Definition: coo.hpp:32
gko::array::get_const_data
const value_type * get_const_data() const noexcept
Returns a constant pointer to the block of memory used to store the elements of the array.
Definition: array.hpp:696
gko::matrix::permute_mode
permute_mode
Specifies how a permutation will be applied to a matrix.
Definition: permutation.hpp:42
gko::matrix::Dense::Dense
Dense(const Dense &)
Copy-constructs a Dense matrix.
gko::matrix::Sellp
SELL-P is a matrix format similar to ELL format.
Definition: csr.hpp:37
gko::make_const_dense_view
std::unique_ptr< const matrix::Dense< typename detail::pointee< VecPtr >::value_type > > make_const_dense_view(VecPtr &&vector)
Creates a view of a given Dense vector.
Definition: dense.hpp:1564
gko::PolymorphicObject::get_executor
std::shared_ptr< const Executor > get_executor() const noexcept
Returns the Executor of the object.
Definition: polymorphic_object.hpp:243
gko::array::get_size
size_type get_size() const noexcept
Returns the number of elements in the array.
Definition: array.hpp:670
gko::LinOp::get_size
const dim< 2 > & get_size() const noexcept
Returns the size of the operator.
Definition: lin_op.hpp:210
gko::local_span
A span that is used exclusively for local numbering.
Definition: range.hpp:138
gko::matrix::Dense::create_submatrix
std::unique_ptr< Dense > create_submatrix(const local_span &rows, const local_span &columns, dim< 2 > size)
Create a submatrix from the original matrix.
Definition: dense.hpp:1160
gko::matrix::Dense::create_real_view
std::unique_ptr< real_type > create_real_view()
Create a real view of the (potentially) complex original matrix.
gko::LinOp::LinOp
LinOp(const LinOp &)=default
Copy-constructs a LinOp.
gko::to_complex
typename detail::to_complex_s< T >::type to_complex
Obtain the type which adds the complex of complex/scalar type or the template parameter of class by a...
Definition: math.hpp:283
gko::matrix::Coo
COO stores a matrix in the coordinate matrix format.
Definition: coo.hpp:50