Ginkgo  Generated from pipelines/2897303314 branch based on develop. Ginkgo version 2.0.0
A numerical linear algebra library targeting many-core architectures
dense.hpp
1 // SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors
2 //
3 // SPDX-License-Identifier: BSD-3-Clause
4 
5 #ifndef GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
6 #define GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
7 
8 
9 #include <initializer_list>
10 #include <type_traits>
11 
12 #include <ginkgo/core/base/array.hpp>
13 #include <ginkgo/core/base/exception_helpers.hpp>
14 #include <ginkgo/core/base/executor.hpp>
15 #include <ginkgo/core/base/lin_op.hpp>
16 #include <ginkgo/core/base/range_accessors.hpp>
17 #include <ginkgo/core/base/types.hpp>
18 #include <ginkgo/core/base/utils.hpp>
19 #include <ginkgo/core/matrix/device_views.hpp>
20 #include <ginkgo/core/matrix/permutation.hpp>
21 #include <ginkgo/core/matrix/scaled_permutation.hpp>
22 
23 
24 namespace gko {
25 namespace experimental {
26 namespace distributed {
27 
28 
29 template <typename ValueType>
30 class Vector;
31 
32 
33 namespace detail {
34 
35 
36 template <typename ValueType>
37 class VectorCache;
38 
39 
40 } // namespace detail
41 } // namespace distributed
42 } // namespace experimental
43 
44 
45 namespace matrix {
46 
47 
48 template <typename ValueType, typename IndexType>
49 class Coo;
50 
51 template <typename ValueType, typename IndexType>
52 class Csr;
53 
54 template <typename ValueType>
55 class Diagonal;
56 
57 template <typename ValueType, typename IndexType>
58 class Ell;
59 
60 template <typename ValueType, typename IndexType>
61 class Fbcsr;
62 
63 template <typename ValueType, typename IndexType>
64 class Hybrid;
65 
66 template <typename ValueType, typename IndexType>
67 class Sellp;
68 
69 template <typename ValueType, typename IndexType>
70 class SparsityCsr;
71 
72 
87 template <typename ValueType = default_precision>
88 class Dense
89  : public LinOp,
90  public EnableCloneable<Dense<ValueType>>,
91  public ConvertibleTo<Dense<next_precision<ValueType>>>,
92 #if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16
93  public ConvertibleTo<Dense<next_precision<ValueType, 2>>>,
94 #endif
95 #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16
96  public ConvertibleTo<Dense<next_precision<ValueType, 3>>>,
97 #endif
98  public ConvertibleTo<Coo<ValueType, int32>>,
99  public ConvertibleTo<Coo<ValueType, int64>>,
100  public ConvertibleTo<Csr<ValueType, int32>>,
101  public ConvertibleTo<Csr<ValueType, int64>>,
102  public ConvertibleTo<Ell<ValueType, int32>>,
103  public ConvertibleTo<Ell<ValueType, int64>>,
104  public ConvertibleTo<Fbcsr<ValueType, int32>>,
105  public ConvertibleTo<Fbcsr<ValueType, int64>>,
106  public ConvertibleTo<Hybrid<ValueType, int32>>,
107  public ConvertibleTo<Hybrid<ValueType, int64>>,
108  public ConvertibleTo<Sellp<ValueType, int32>>,
109  public ConvertibleTo<Sellp<ValueType, int64>>,
110  public ConvertibleTo<SparsityCsr<ValueType, int32>>,
111  public ConvertibleTo<SparsityCsr<ValueType, int64>>,
112  public DiagonalExtractable<ValueType>,
113  public ReadableFromMatrixData<ValueType, int32>,
114  public ReadableFromMatrixData<ValueType, int64>,
115  public WritableToMatrixData<ValueType, int32>,
116  public WritableToMatrixData<ValueType, int64>,
117  public Transposable,
118  public Permutable<int32>,
119  public Permutable<int64>,
120  public EnableAbsoluteComputation<remove_complex<Dense<ValueType>>>,
121  public ScaledIdentityAddable {
122  friend class EnableCloneable<Dense>;
123  friend class Coo<ValueType, int32>;
124  friend class Coo<ValueType, int64>;
125  friend class Csr<ValueType, int32>;
126  friend class Csr<ValueType, int64>;
127  friend class Diagonal<ValueType>;
128  friend class Ell<ValueType, int32>;
129  friend class Ell<ValueType, int64>;
130  friend class Fbcsr<ValueType, int32>;
131  friend class Fbcsr<ValueType, int64>;
132  friend class Hybrid<ValueType, int32>;
133  friend class Hybrid<ValueType, int64>;
134  friend class Sellp<ValueType, int32>;
135  friend class Sellp<ValueType, int64>;
136  friend class SparsityCsr<ValueType, int32>;
137  friend class SparsityCsr<ValueType, int64>;
138  friend class Dense<to_complex<ValueType>>;
139  friend class experimental::distributed::Vector<ValueType>;
140  friend class experimental::distributed::detail::VectorCache<ValueType>;
141  GKO_ASSERT_SUPPORTED_VALUE_TYPE;
142 
143 public:
146  using ConvertibleTo<Dense<next_precision<ValueType>>>::convert_to;
147  using ConvertibleTo<Dense<next_precision<ValueType>>>::move_to;
148  using ConvertibleTo<Coo<ValueType, int32>>::convert_to;
149  using ConvertibleTo<Coo<ValueType, int32>>::move_to;
150  using ConvertibleTo<Coo<ValueType, int64>>::convert_to;
151  using ConvertibleTo<Coo<ValueType, int64>>::move_to;
152  using ConvertibleTo<Csr<ValueType, int32>>::convert_to;
153  using ConvertibleTo<Csr<ValueType, int32>>::move_to;
154  using ConvertibleTo<Csr<ValueType, int64>>::convert_to;
155  using ConvertibleTo<Csr<ValueType, int64>>::move_to;
156  using ConvertibleTo<Ell<ValueType, int32>>::convert_to;
157  using ConvertibleTo<Ell<ValueType, int32>>::move_to;
158  using ConvertibleTo<Ell<ValueType, int64>>::convert_to;
159  using ConvertibleTo<Ell<ValueType, int64>>::move_to;
160  using ConvertibleTo<Fbcsr<ValueType, int32>>::convert_to;
161  using ConvertibleTo<Fbcsr<ValueType, int32>>::move_to;
162  using ConvertibleTo<Fbcsr<ValueType, int64>>::convert_to;
163  using ConvertibleTo<Fbcsr<ValueType, int64>>::move_to;
164  using ConvertibleTo<Hybrid<ValueType, int32>>::convert_to;
165  using ConvertibleTo<Hybrid<ValueType, int32>>::move_to;
166  using ConvertibleTo<Hybrid<ValueType, int64>>::convert_to;
167  using ConvertibleTo<Hybrid<ValueType, int64>>::move_to;
168  using ConvertibleTo<Sellp<ValueType, int32>>::convert_to;
169  using ConvertibleTo<Sellp<ValueType, int32>>::move_to;
170  using ConvertibleTo<Sellp<ValueType, int64>>::convert_to;
171  using ConvertibleTo<Sellp<ValueType, int64>>::move_to;
172  using ConvertibleTo<SparsityCsr<ValueType, int32>>::convert_to;
173  using ConvertibleTo<SparsityCsr<ValueType, int32>>::move_to;
174  using ConvertibleTo<SparsityCsr<ValueType, int64>>::convert_to;
175  using ConvertibleTo<SparsityCsr<ValueType, int64>>::move_to;
178 
179  using value_type = ValueType;
180  using index_type = int64;
181  using transposed_type = Dense<value_type>;
182  using mat_data = matrix_data<value_type, int64>;
183  using mat_data32 = matrix_data<value_type, int32>;
184  using device_mat_data = device_matrix_data<value_type, int64>;
185  using device_mat_data32 = device_matrix_data<value_type, int32>;
186  using absolute_type = remove_complex<Dense>;
187  using real_type = absolute_type;
188  using complex_type = to_complex<Dense>;
189  using device_view = matrix::view::dense<value_type>;
190  using const_device_view = matrix::view::dense<const value_type>;
191 
192  using row_major_range = gko::range<gko::accessor::row_major<ValueType, 2>>;
193 
200  static std::unique_ptr<Dense> create_with_config_of(
202  {
203  // De-referencing `other` before calling the functions (instead of
204  // using operator `->`) is currently required to be compatible with
205  // CUDA 10.1.
206  // Otherwise, it results in a compile error.
207  return (*other).create_with_same_config();
208  }
209 
221  static std::unique_ptr<Dense> create_with_type_of(
222  ptr_param<const Dense> other, std::shared_ptr<const Executor> exec,
223  const dim<2>& size = dim<2>{})
224  {
225  // See create_with_config_of()
226  return (*other).create_with_type_of_impl(exec, size, size[1]);
227  }
228 
237  static std::unique_ptr<Dense> create_with_type_of(
238  ptr_param<const Dense> other, std::shared_ptr<const Executor> exec,
239  const dim<2>& size, size_type stride)
240  {
241  // See create_with_config_of()
242  return (*other).create_with_type_of_impl(exec, size, stride);
243  }
244 
255  static std::unique_ptr<Dense> create_with_type_of(
256  ptr_param<const Dense> other, std::shared_ptr<const Executor> exec,
257  const dim<2>& size, const dim<2>& local_size, size_type stride)
258  {
259  // See create_with_config_of()
260  return (*other).create_with_type_of_impl(exec, size, stride);
261  }
262 
271  static std::unique_ptr<Dense> create_view_of(ptr_param<Dense> other)
272  {
273  return other->create_view_of_impl();
274  }
275 
283  static std::unique_ptr<const Dense> create_const_view_of(
285  {
286  return other->create_const_view_of_impl();
287  }
288 
289  friend class Dense<previous_precision<ValueType>>;
290 
291  void convert_to(Dense<next_precision<ValueType>>* result) const override;
292 
293  void move_to(Dense<next_precision<ValueType>>* result) override;
294 
295 #if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16
296  friend class Dense<previous_precision<ValueType, 2>>;
299 
300  void convert_to(Dense<next_precision<ValueType, 2>>* result) const override;
301 
302  void move_to(Dense<next_precision<ValueType, 2>>* result) override;
303 #endif
304 
305 #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16
306  friend class Dense<previous_precision<ValueType, 3>>;
309 
310  void convert_to(Dense<next_precision<ValueType, 3>>* result) const override;
311 
312  void move_to(Dense<next_precision<ValueType, 3>>* result) override;
313 #endif
314 
315  void convert_to(Coo<ValueType, int32>* result) const override;
316 
317  void move_to(Coo<ValueType, int32>* result) override;
318 
319  void convert_to(Coo<ValueType, int64>* result) const override;
320 
321  void move_to(Coo<ValueType, int64>* result) override;
322 
323  void convert_to(Csr<ValueType, int32>* result) const override;
324 
325  void move_to(Csr<ValueType, int32>* result) override;
326 
327  void convert_to(Csr<ValueType, int64>* result) const override;
328 
329  void move_to(Csr<ValueType, int64>* result) override;
330 
331  void convert_to(Ell<ValueType, int32>* result) const override;
332 
333  void move_to(Ell<ValueType, int32>* result) override;
334 
335  void convert_to(Ell<ValueType, int64>* result) const override;
336 
337  void move_to(Ell<ValueType, int64>* result) override;
338 
339  void convert_to(Fbcsr<ValueType, int32>* result) const override;
340 
341  void move_to(Fbcsr<ValueType, int32>* result) override;
342 
343  void convert_to(Fbcsr<ValueType, int64>* result) const override;
344 
345  void move_to(Fbcsr<ValueType, int64>* result) override;
346 
347  void convert_to(Hybrid<ValueType, int32>* result) const override;
348 
349  void move_to(Hybrid<ValueType, int32>* result) override;
350 
351  void convert_to(Hybrid<ValueType, int64>* result) const override;
352 
353  void move_to(Hybrid<ValueType, int64>* result) override;
354 
355  void convert_to(Sellp<ValueType, int32>* result) const override;
356 
357  void move_to(Sellp<ValueType, int32>* result) override;
358 
359  void convert_to(Sellp<ValueType, int64>* result) const override;
360 
361  void move_to(Sellp<ValueType, int64>* result) override;
362 
363  void convert_to(SparsityCsr<ValueType, int32>* result) const override;
364 
365  void move_to(SparsityCsr<ValueType, int32>* result) override;
366 
367  void convert_to(SparsityCsr<ValueType, int64>* result) const override;
368 
369  void move_to(SparsityCsr<ValueType, int64>* result) override;
370 
371  void read(const mat_data& data) override;
372 
373  void read(const mat_data32& data) override;
374 
375  void read(const device_mat_data& data) override;
376 
377  void read(const device_mat_data32& data) override;
378 
379  void read(device_mat_data&& data) override;
380 
381  void read(device_mat_data32&& data) override;
382 
385 
386  void write(mat_data& data) const override;
387 
388  void write(mat_data32& data) const override;
389 
390  std::unique_ptr<LinOp> transpose() const override;
391 
392  std::unique_ptr<LinOp> conj_transpose() const override;
393 
400  void transpose(ptr_param<Dense> output) const;
401 
408  void conj_transpose(ptr_param<Dense> output) const;
409 
415  void fill(const ValueType value);
416 
427  std::unique_ptr<Dense> permute(
428  ptr_param<const Permutation<int32>> permutation,
430 
434  std::unique_ptr<Dense> permute(
435  ptr_param<const Permutation<int64>> permutation,
437 
445  void permute(ptr_param<const Permutation<int32>> permutation,
446  ptr_param<Dense> output, permute_mode mode) const;
447 
452  void permute(ptr_param<const Permutation<int64>> permutation,
453  ptr_param<Dense> output, permute_mode mode) const;
454 
469  std::unique_ptr<Dense> permute(
470  ptr_param<const Permutation<int32>> row_permutation,
471  ptr_param<const Permutation<int32>> column_permutation,
472  bool invert = false) const;
473 
478  std::unique_ptr<Dense> permute(
479  ptr_param<const Permutation<int64>> row_permutation,
480  ptr_param<const Permutation<int64>> column_permutation,
481  bool invert = false) const;
482 
492  void permute(ptr_param<const Permutation<int32>> row_permutation,
493  ptr_param<const Permutation<int32>> column_permutation,
494  ptr_param<Dense> output, bool invert = false) const;
495 
500  void permute(ptr_param<const Permutation<int64>> row_permutation,
501  ptr_param<const Permutation<int64>> column_permutation,
502  ptr_param<Dense> output, bool invert = false) const;
503 
513  std::unique_ptr<Dense> scale_permute(
516 
521  std::unique_ptr<Dense> scale_permute(
524 
533  void scale_permute(
535  ptr_param<Dense> output, permute_mode mode) const;
536 
541  void scale_permute(
543  ptr_param<Dense> output, permute_mode mode) const;
544 
557  std::unique_ptr<Dense> scale_permute(
558  ptr_param<const ScaledPermutation<value_type, int32>> row_permutation,
560  column_permutation,
561  bool invert = false) const;
562 
567  std::unique_ptr<Dense> scale_permute(
568  ptr_param<const ScaledPermutation<value_type, int64>> row_permutation,
570  column_permutation,
571  bool invert = false) const;
572 
582  void scale_permute(
583  ptr_param<const ScaledPermutation<value_type, int32>> row_permutation,
585  column_permutation,
586  ptr_param<Dense> output, bool invert = false) const;
587 
593  void scale_permute(
594  ptr_param<const ScaledPermutation<value_type, int64>> row_permutation,
596  column_permutation,
597  ptr_param<Dense> output, bool invert = false) const;
598 
599  std::unique_ptr<LinOp> permute(
600  const array<int32>* permutation_indices) const override;
601 
602  std::unique_ptr<LinOp> permute(
603  const array<int64>* permutation_indices) const override;
604 
614  void permute(const array<int32>* permutation_indices,
615  ptr_param<Dense> output) const;
616 
620  void permute(const array<int64>* permutation_indices,
621  ptr_param<Dense> output) const;
622 
623  std::unique_ptr<LinOp> inverse_permute(
624  const array<int32>* permutation_indices) const override;
625 
626  std::unique_ptr<LinOp> inverse_permute(
627  const array<int64>* permutation_indices) const override;
628 
639  void inverse_permute(const array<int32>* permutation_indices,
640  ptr_param<Dense> output) const;
641 
645  void inverse_permute(const array<int64>* permutation_indices,
646  ptr_param<Dense> output) const;
647 
648  std::unique_ptr<LinOp> row_permute(
649  const array<int32>* permutation_indices) const override;
650 
651  std::unique_ptr<LinOp> row_permute(
652  const array<int64>* permutation_indices) const override;
653 
663  void row_permute(const array<int32>* permutation_indices,
664  ptr_param<Dense> output) const;
665 
669  void row_permute(const array<int64>* permutation_indices,
670  ptr_param<Dense> output) const;
671 
682  std::unique_ptr<Dense> row_gather(const array<int32>* gather_indices) const;
683 
687  std::unique_ptr<Dense> row_gather(const array<int64>* gather_indices) const;
688 
701  void row_gather(const array<int32>* gather_indices,
702  ptr_param<LinOp> row_collection) const;
703 
707  void row_gather(const array<int64>* gather_indices,
708  ptr_param<LinOp> row_collection) const;
709 
724  const array<int32>* gather_indices,
726  ptr_param<LinOp> row_collection) const;
727 
733  const array<int64>* gather_indices,
735  ptr_param<LinOp> row_collection) const;
736 
737  std::unique_ptr<LinOp> column_permute(
738  const array<int32>* permutation_indices) const override;
739 
740  std::unique_ptr<LinOp> column_permute(
741  const array<int64>* permutation_indices) const override;
742 
752  void column_permute(const array<int32>* permutation_indices,
753  ptr_param<Dense> output) const;
754 
758  void column_permute(const array<int64>* permutation_indices,
759  ptr_param<Dense> output) const;
760 
761  std::unique_ptr<LinOp> inverse_row_permute(
762  const array<int32>* permutation_indices) const override;
763 
764  std::unique_ptr<LinOp> inverse_row_permute(
765  const array<int64>* permutation_indices) const override;
766 
776  void inverse_row_permute(const array<int32>* permutation_indices,
777  ptr_param<Dense> output) const;
778 
782  void inverse_row_permute(const array<int64>* permutation_indices,
783  ptr_param<Dense> output) const;
784 
785  std::unique_ptr<LinOp> inverse_column_permute(
786  const array<int32>* permutation_indices) const override;
787 
788  std::unique_ptr<LinOp> inverse_column_permute(
789  const array<int64>* permutation_indices) const override;
790 
800  void inverse_column_permute(const array<int32>* permutation_indices,
801  ptr_param<Dense> output) const;
802 
806  void inverse_column_permute(const array<int64>* permutation_indices,
807  ptr_param<Dense> output) const;
808 
809  std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
810 
818  void extract_diagonal(ptr_param<Diagonal<ValueType>> output) const;
819 
820  std::unique_ptr<absolute_type> compute_absolute() const override;
821 
829  void compute_absolute(ptr_param<absolute_type> output) const;
830 
831  void compute_absolute_inplace() override;
832 
837  std::unique_ptr<complex_type> make_complex() const;
838 
844  void make_complex(ptr_param<complex_type> result) const;
845 
850  std::unique_ptr<real_type> get_real() const;
851 
855  void get_real(ptr_param<real_type> result) const;
856 
861  std::unique_ptr<real_type> get_imag() const;
862 
867  void get_imag(ptr_param<real_type> result) const;
868 
874  value_type* get_values() noexcept { return values_.get_data(); }
875 
883  const value_type* get_const_values() const noexcept
884  {
885  return values_.get_const_data();
886  }
887 
893  size_type get_stride() const noexcept { return stride_; }
894 
901  {
902  return values_.get_size();
903  }
904 
905  device_view get_device_view();
906 
907  const_device_view get_const_device_view() const;
908 
919  value_type& at(size_type row, size_type col) noexcept
920  {
921  return values_.get_data()[linearize_index(row, col)];
922  }
923 
927  value_type at(size_type row, size_type col) const noexcept
928  {
929  return values_.get_const_data()[linearize_index(row, col)];
930  }
931 
946  ValueType& at(size_type idx) noexcept
947  {
948  return values_.get_data()[linearize_index(idx)];
949  }
950 
954  ValueType at(size_type idx) const noexcept
955  {
956  return values_.get_const_data()[linearize_index(idx)];
957  }
958 
968  void scale(ptr_param<const LinOp> alpha);
969 
979  void inv_scale(ptr_param<const LinOp> alpha);
980 
981  void validate_data() const override;
982 
994 
1006 
1015  void compute_dot(ptr_param<const LinOp> b, ptr_param<LinOp> result) const;
1016 
1029  array<char>& tmp) const;
1030 
1040  ptr_param<LinOp> result) const;
1041 
1054  array<char>& tmp) const;
1055 
1063  void compute_norm2(ptr_param<LinOp> result) const;
1064 
1075  void compute_norm2(ptr_param<LinOp> result, array<char>& tmp) const;
1076 
1084  void compute_norm1(ptr_param<LinOp> result) const;
1085 
1096  void compute_norm1(ptr_param<LinOp> result, array<char>& tmp) const;
1097 
1106  void compute_squared_norm2(ptr_param<LinOp> result) const;
1107 
1119  void compute_squared_norm2(ptr_param<LinOp> result, array<char>& tmp) const;
1120 
1128  void compute_mean(ptr_param<LinOp> result) const;
1129 
1140  void compute_mean(ptr_param<LinOp> result, array<char>& tmp) const;
1141 
1152  std::unique_ptr<Dense> create_submatrix(const span& rows,
1153  const span& columns,
1154  const size_type stride)
1155  {
1156  return this->create_submatrix_impl(rows, columns, stride);
1157  }
1158 
1165  std::unique_ptr<Dense> create_submatrix(const span& rows,
1166  const span& columns)
1167  {
1168  return create_submatrix(rows, columns, this->get_stride());
1169  }
1170 
1171 
1180  std::unique_ptr<Dense> create_submatrix(const local_span& rows,
1181  const local_span& columns,
1182  dim<2> size)
1183  {
1184  dim<2> deduced_size{rows.length(), columns.length()};
1185  GKO_ASSERT_EQUAL_DIMENSIONS(deduced_size, size);
1186  return create_submatrix(rows, columns, this->get_stride());
1187  }
1188 
1196  std::unique_ptr<real_type> create_real_view();
1197 
1201  std::unique_ptr<const real_type> create_real_view() const;
1202 
1215  static std::unique_ptr<Dense> create(std::shared_ptr<const Executor> exec,
1216  const dim<2>& size = {},
1217  size_type stride = 0);
1218 
1235  static std::unique_ptr<Dense> create(std::shared_ptr<const Executor> exec,
1236  const dim<2>& size,
1237  array<value_type> values,
1238  size_type stride);
1239 
1244  template <typename InputValueType>
1245  GKO_DEPRECATED(
1246  "explicitly construct the gko::array argument instead of passing an"
1247  "initializer list")
1248  static std::unique_ptr<Dense> create(
1249  std::shared_ptr<const Executor> exec, const dim<2>& size,
1250  std::initializer_list<InputValueType> values, size_type stride)
1251  {
1252  return create(exec, size, array<value_type>{exec, std::move(values)},
1253  stride);
1254  }
1255 
1267  static std::unique_ptr<const Dense> create_const(
1268  std::shared_ptr<const Executor> exec, const dim<2>& size,
1269  gko::detail::const_array_view<ValueType>&& values, size_type stride);
1270 
1276  Dense& operator=(const Dense&);
1277 
1283  Dense& operator=(Dense&&);
1284 
1289  Dense(const Dense&);
1290 
1295  Dense(Dense&&);
1296 
1297 protected:
1298  Dense(std::shared_ptr<const Executor> exec, const dim<2>& size = {},
1299  size_type stride = 0);
1300 
1301  Dense(std::shared_ptr<const Executor> exec, const dim<2>& size,
1302  array<value_type> values, size_type stride);
1303 
1310  virtual std::unique_ptr<Dense> create_with_same_config() const
1311  {
1312  return Dense::create(this->get_executor(), this->get_size(),
1313  this->get_stride());
1314  }
1315 
1323  virtual std::unique_ptr<Dense> create_with_type_of_impl(
1324  std::shared_ptr<const Executor> exec, const dim<2>& size,
1325  size_type stride) const
1326  {
1327  return Dense::create(exec, size, stride);
1328  }
1329 
1336  virtual std::unique_ptr<Dense> create_view_of_impl()
1337  {
1338  auto exec = this->get_executor();
1339  return Dense::create(
1340  exec, this->get_size(),
1342  this->get_values()),
1343  this->get_stride());
1344  }
1345 
1352  virtual std::unique_ptr<const Dense> create_const_view_of_impl() const
1353  {
1354  auto exec = this->get_executor();
1355  return Dense::create_const(
1356  exec, this->get_size(),
1358  this->get_const_values()),
1359  this->get_stride());
1360  }
1361 
1362  template <typename IndexType>
1363  void convert_impl(Coo<ValueType, IndexType>* result) const;
1364 
1365  template <typename IndexType>
1366  void convert_impl(Csr<ValueType, IndexType>* result) const;
1367 
1368  template <typename IndexType>
1369  void convert_impl(Ell<ValueType, IndexType>* result) const;
1370 
1371  template <typename IndexType>
1372  void convert_impl(Fbcsr<ValueType, IndexType>* result) const;
1373 
1374  template <typename IndexType>
1375  void convert_impl(Hybrid<ValueType, IndexType>* result) const;
1376 
1377  template <typename IndexType>
1378  void convert_impl(Sellp<ValueType, IndexType>* result) const;
1379 
1380  template <typename IndexType>
1381  void convert_impl(SparsityCsr<ValueType, IndexType>* result) const;
1382 
1389  virtual void scale_impl(const LinOp* alpha);
1390 
1397  virtual void inv_scale_impl(const LinOp* alpha);
1398 
1405  virtual void add_scaled_impl(const LinOp* alpha, const LinOp* b);
1406 
1413  virtual void sub_scaled_impl(const LinOp* alpha, const LinOp* b);
1414 
1421  virtual void compute_dot_impl(const LinOp* b, LinOp* result) const;
1422 
1429  virtual void compute_conj_dot_impl(const LinOp* b, LinOp* result) const;
1430 
1437  virtual void compute_norm2_impl(LinOp* result) const;
1438 
1445  virtual void compute_norm1_impl(LinOp* result) const;
1446 
1453  virtual void compute_squared_norm2_impl(LinOp* result) const;
1454 
1458  virtual void compute_mean_impl(LinOp* result) const;
1459 
1468  void resize(gko::dim<2> new_size);
1469 
1477  virtual std::unique_ptr<Dense> create_submatrix_impl(
1478  const span& rows, const span& columns, const size_type stride);
1479 
1480  void apply_impl(const LinOp* b, LinOp* x) const override;
1481 
1482  void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta,
1483  LinOp* x) const override;
1484 
1485  size_type linearize_index(size_type row, size_type col) const noexcept
1486  {
1487  return row * stride_ + col;
1488  }
1489 
1490  size_type linearize_index(size_type idx) const noexcept
1491  {
1492  return linearize_index(idx / this->get_size()[1],
1493  idx % this->get_size()[1]);
1494  }
1495 
1496  template <typename IndexType>
1497  void permute_impl(const Permutation<IndexType>* permutation,
1498  permute_mode mode, Dense* output) const;
1499 
1500  template <typename IndexType>
1501  void permute_impl(const Permutation<IndexType>* row_permutation,
1502  const Permutation<IndexType>* col_permutation,
1503  bool invert, Dense* output) const;
1504 
1505  template <typename IndexType>
1506  void scale_permute_impl(
1507  const ScaledPermutation<ValueType, IndexType>* permutation,
1508  permute_mode mode, Dense* output) const;
1509 
1510  template <typename IndexType>
1511  void scale_permute_impl(
1512  const ScaledPermutation<ValueType, IndexType>* row_permutation,
1513  const ScaledPermutation<ValueType, IndexType>* column_permutation,
1514  bool invert, Dense* output) const;
1515 
1516  template <typename OutputType, typename IndexType>
1517  void row_gather_impl(const array<IndexType>* row_idxs,
1518  Dense<OutputType>* row_collection) const;
1519 
1520  template <typename OutputType, typename IndexType>
1521  void row_gather_impl(const Dense<ValueType>* alpha,
1522  const array<IndexType>* row_idxs,
1523  const Dense<ValueType>* beta,
1524  Dense<OutputType>* row_collection) const;
1525 
1526 private:
1527  size_type stride_;
1528  array<value_type> values_;
1529 
1530  void add_scaled_identity_impl(const LinOp* a, const LinOp* b) override;
1531 };
1532 
1533 
1534 } // namespace matrix
1535 
1536 
1537 namespace detail {
1538 
1539 
1540 template <typename ValueType>
1541 struct temporary_clone_helper<matrix::Dense<ValueType>> {
1542  static std::unique_ptr<matrix::Dense<ValueType>> create(
1543  std::shared_ptr<const Executor> exec, matrix::Dense<ValueType>* ptr,
1544  bool copy_data)
1545  {
1546  if (copy_data) {
1547  return gko::clone(std::move(exec), ptr);
1548  } else {
1549  return matrix::Dense<ValueType>::create(exec, ptr->get_size());
1550  }
1551  }
1552 };
1553 
1554 
1555 } // namespace detail
1556 
1557 
1565 template <typename VecPtr>
1566 std::unique_ptr<matrix::Dense<typename detail::pointee<VecPtr>::value_type>>
1567 make_dense_view(VecPtr&& vector)
1568 {
1569  using value_type = typename detail::pointee<VecPtr>::value_type;
1571 }
1572 
1573 
1581 template <typename VecPtr>
1582 std::unique_ptr<
1583  const matrix::Dense<typename detail::pointee<VecPtr>::value_type>>
1584 make_const_dense_view(VecPtr&& vector)
1585 {
1586  using value_type = typename detail::pointee<VecPtr>::value_type;
1588 }
1589 
1590 
1611 template <typename Matrix, typename... TArgs>
1612 std::unique_ptr<Matrix> initialize(
1613  size_type stride, std::initializer_list<typename Matrix::value_type> vals,
1614  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1615 {
1617  size_type num_rows = vals.size();
1618  auto tmp = dense::create(exec->get_master(), dim<2>{num_rows, 1}, stride);
1619  size_type idx = 0;
1620  for (const auto& elem : vals) {
1621  tmp->at(idx) = elem;
1622  ++idx;
1623  }
1624  auto mtx = Matrix::create(exec, std::forward<TArgs>(create_args)...);
1625  tmp->move_to(mtx);
1626  return mtx;
1627 }
1628 
1649 template <typename Matrix, typename... TArgs>
1650 std::unique_ptr<Matrix> initialize(
1651  std::initializer_list<typename Matrix::value_type> vals,
1652  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1653 {
1654  return initialize<Matrix>(1, vals, std::move(exec),
1655  std::forward<TArgs>(create_args)...);
1656 }
1657 
1658 
1679 template <typename Matrix, typename... TArgs>
1680 std::unique_ptr<Matrix> initialize(
1681  size_type stride,
1682  std::initializer_list<std::initializer_list<typename Matrix::value_type>>
1683  vals,
1684  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1685 {
1687  size_type num_rows = vals.size();
1688  size_type num_cols = num_rows > 0 ? begin(vals)->size() : 1;
1689  auto tmp =
1690  dense::create(exec->get_master(), dim<2>{num_rows, num_cols}, stride);
1691  size_type ridx = 0;
1692  for (const auto& row : vals) {
1693  size_type cidx = 0;
1694  for (const auto& elem : row) {
1695  tmp->at(ridx, cidx) = elem;
1696  ++cidx;
1697  }
1698  ++ridx;
1699  }
1700  auto mtx = Matrix::create(exec, std::forward<TArgs>(create_args)...);
1701  tmp->move_to(mtx);
1702  return mtx;
1703 }
1704 
1705 
1727 template <typename Matrix, typename... TArgs>
1728 std::unique_ptr<Matrix> initialize(
1729  std::initializer_list<std::initializer_list<typename Matrix::value_type>>
1730  vals,
1731  std::shared_ptr<const Executor> exec, TArgs&&... create_args)
1732 {
1733  return initialize<Matrix>(vals.size() > 0 ? begin(vals)->size() : 0, vals,
1734  std::move(exec),
1735  std::forward<TArgs>(create_args)...);
1736 }
1737 
1738 
1739 } // namespace gko
1740 
1741 
1742 #endif // GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
gko::matrix::Dense::permute
std::unique_ptr< Dense > permute(ptr_param< const Permutation< int32 >> permutation, permute_mode mode=permute_mode::symmetric) const
Creates a permuted copy of this matrix with the given permutation .
gko::matrix::Dense::inverse_column_permute
std::unique_ptr< LinOp > inverse_column_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the row permutation of the inverse permuted object.
gko::matrix::Dense::add_scaled
void add_scaled(ptr_param< const LinOp > alpha, ptr_param< const LinOp > b)
Adds b scaled by alpha to the matrix (aka: BLAS axpy).
gko::matrix::Fbcsr
Fixed-block compressed sparse row storage matrix format.
Definition: csr.hpp:47
gko::matrix::Csr
CSR is a matrix format which stores only the nonzero coefficients by compressing each row of the matr...
Definition: matrix.hpp:30
gko::matrix::permute_mode::rows
The rows will be permuted.
gko::matrix::Dense::transpose
std::unique_ptr< LinOp > transpose() const override
Returns a LinOp representing the transpose of the Transposable object.
gko::matrix::permute_mode::columns
The columns will be permuted.
gko::ReadableFromMatrixData< ValueType, int32 >::read
virtual void read(const matrix_data< ValueType, int32 > &data)=0
Reads a matrix from a matrix_data structure.
gko::matrix::Dense
Dense is a matrix format which explicitly stores all values of the matrix.
Definition: dense_cache.hpp:28
gko::matrix::SparsityCsr
SparsityCsr is a matrix format which stores only the sparsity pattern of a sparse matrix by compressi...
Definition: csr.hpp:41
gko::matrix::Dense::at
ValueType at(size_type idx) const noexcept
Returns a single element of the matrix.
Definition: dense.hpp:954
gko::matrix::Dense::row_gather
std::unique_ptr< Dense > row_gather(const array< int32 > *gather_indices) const
Create a Dense matrix consisting of the given rows from this matrix.
gko::matrix::Dense::make_complex
std::unique_ptr< complex_type > make_complex() const
Creates a complex copy of the original matrix.
gko::matrix::Dense::create
static std::unique_ptr< Dense > create(std::shared_ptr< const Executor > exec, const dim< 2 > &size={}, size_type stride=0)
Creates an uninitialized Dense matrix of the specified size.
gko::matrix::Dense::create_with_type_of
static std::unique_ptr< Dense > create_with_type_of(ptr_param< const Dense > other, std::shared_ptr< const Executor > exec, const dim< 2 > &size, size_type stride)
Definition: dense.hpp:237
gko::matrix::Dense::scale
void scale(ptr_param< const LinOp > alpha)
Scales the matrix with a scalar (aka: BLAS scal).
gko::matrix::ScaledPermutation
ScaledPermutation is a matrix combining a permutation with scaling factors.
Definition: scaled_permutation.hpp:35
gko::size_type
std::size_t size_type
Integral type used for allocation quantities.
Definition: types.hpp:101
gko::matrix::Dense::get_real
std::unique_ptr< real_type > get_real() const
Creates a new real matrix and extracts the real part of the original matrix into that.
gko::matrix::Dense::inv_scale
void inv_scale(ptr_param< const LinOp > alpha)
Scales the matrix with the inverse of a scalar.
gko::matrix::Permutation
Permutation is a matrix format that represents a permutation matrix, i.e.
Definition: permutation.hpp:110
gko::matrix::Dense::fill
void fill(const ValueType value)
Fill the dense matrix with a given value.
gko::make_const_array_view
detail::const_array_view< ValueType > make_const_array_view(std::shared_ptr< const Executor > exec, size_type size, const ValueType *data)
Helper function to create a const array view deducing the value type.
Definition: array.hpp:819
gko::matrix::Dense::compute_dot
void compute_dot(ptr_param< const LinOp > b, ptr_param< LinOp > result) const
Computes the column-wise dot product of this matrix and b.
gko::matrix::Dense::create_submatrix
std::unique_ptr< Dense > create_submatrix(const span &rows, const span &columns, const size_type stride)
Create a submatrix from the original matrix.
Definition: dense.hpp:1152
gko::matrix::Dense::create_with_config_of
static std::unique_ptr< Dense > create_with_config_of(ptr_param< const Dense > other)
Creates a Dense matrix with the same size and stride as another Dense matrix.
Definition: dense.hpp:200
gko::initialize
std::unique_ptr< Matrix > initialize(size_type stride, std::initializer_list< typename Matrix::value_type > vals, std::shared_ptr< const Executor > exec, TArgs &&... create_args)
Creates and initializes a column-vector.
Definition: dense.hpp:1612
gko::clone
detail::cloned_type< Pointer > clone(const Pointer &p)
Creates a unique clone of the object pointed to by p.
Definition: utils_helper.hpp:425
gko::matrix::Dense::at
value_type & at(size_type row, size_type col) noexcept
Returns a single element of the matrix.
Definition: dense.hpp:919
gko::matrix::Dense::get_stride
size_type get_stride() const noexcept
Returns the stride of the matrix.
Definition: dense.hpp:893
gko::matrix::Dense::create_view_of
static std::unique_ptr< Dense > create_view_of(ptr_param< Dense > other)
Creates a Dense matrix, where the underlying array is a view of another Dense matrix' array.
Definition: dense.hpp:271
gko::range
A range is a multidimensional view of the memory.
Definition: range.hpp:302
gko
The Ginkgo namespace.
Definition: abstract_factory.hpp:19
gko::matrix::Dense::get_num_stored_elements
size_type get_num_stored_elements() const noexcept
Returns the number of elements explicitly stored in the matrix.
Definition: dense.hpp:900
gko::matrix::Dense::sub_scaled
void sub_scaled(ptr_param< const LinOp > alpha, ptr_param< const LinOp > b)
Subtracts b scaled by alpha from the matrix (aka: BLAS axpy).
gko::matrix::Dense::get_values
value_type * get_values() noexcept
Returns a pointer to the array of values of the matrix.
Definition: dense.hpp:874
gko::array
An array is a container which encapsulates fixed-sized arrays, stored on the Executor tied to the arr...
Definition: array.hpp:26
gko::matrix::Dense::compute_norm2
void compute_norm2(ptr_param< LinOp > result) const
Computes the column-wise Euclidean (L^2) norm of this matrix.
gko::matrix::Dense::row_permute
std::unique_ptr< LinOp > row_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the row permutation of the Permutable object.
gko::matrix::Dense::create_with_type_of
static std::unique_ptr< Dense > create_with_type_of(ptr_param< const Dense > other, std::shared_ptr< const Executor > exec, const dim< 2 > &size=dim< 2 >{})
Creates a Dense matrix with the same type as another Dense matrix but on a different executor and wit...
Definition: dense.hpp:221
gko::span
A span is a lightweight structure used to create sub-ranges from other ranges.
Definition: range.hpp:45
gko::dim< 2 >
gko::matrix::Diagonal
This class is a utility which efficiently implements the diagonal matrix (a linear operator which sca...
Definition: lin_op.hpp:31
gko::matrix::Dense::create_const
static std::unique_ptr< const Dense > create_const(std::shared_ptr< const Executor > exec, const dim< 2 > &size, gko::detail::const_array_view< ValueType > &&values, size_type stride)
Creates a constant (immutable) Dense matrix from a constant array.
gko::ptr_param
This class is used for function parameters in the place of raw pointers.
Definition: utils_helper.hpp:43
gko::matrix::Dense::at
value_type at(size_type row, size_type col) const noexcept
Returns a single element of the matrix.
Definition: dense.hpp:927
gko::array::get_data
value_type * get_data() noexcept
Returns a pointer to the block of memory used to store the elements of the array.
Definition: array.hpp:686
gko::matrix::Dense::validate_data
void validate_data() const override
Throws gko::InvalidData exception if we found the data inside the object does not fulfill certain pro...
gko::matrix::Dense::compute_mean
void compute_mean(ptr_param< LinOp > result) const
Computes the column-wise arithmetic mean of this matrix.
gko::matrix::Dense::inverse_row_permute
std::unique_ptr< LinOp > inverse_row_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the row permutation of the inverse permuted object.
gko::WritableToMatrixData
A LinOp implementing this interface can write its data to a matrix_data structure.
Definition: lin_op.hpp:619
gko::matrix::permute_mode::symmetric
The rows and columns will be permuted.
gko::matrix::Dense::get_imag
std::unique_ptr< real_type > get_imag() const
Creates a new real matrix and extracts the imaginary part of the original matrix into that.
gko::matrix::Dense::compute_squared_norm2
void compute_squared_norm2(ptr_param< LinOp > result) const
Computes the square of the column-wise Euclidean (L^2) norm of this matrix.
gko::matrix::Dense::operator=
Dense & operator=(const Dense &)
Copy-assigns a Dense matrix.
gko::matrix::Dense::inverse_permute
std::unique_ptr< LinOp > inverse_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the symmetric inverse row and column permutation of the Permutable objec...
gko::matrix::Dense::create_with_type_of
static std::unique_ptr< Dense > create_with_type_of(ptr_param< const Dense > other, std::shared_ptr< const Executor > exec, const dim< 2 > &size, const dim< 2 > &local_size, size_type stride)
Definition: dense.hpp:255
gko::stop::mode
mode
The mode for the residual norm criterion.
Definition: residual_norm.hpp:37
gko::matrix::Dense::scale_permute
std::unique_ptr< Dense > scale_permute(ptr_param< const ScaledPermutation< value_type, int32 >> permutation, permute_mode mode=permute_mode::symmetric) const
Creates a scaled and permuted copy of this matrix.
gko::matrix::Dense::compute_norm1
void compute_norm1(ptr_param< LinOp > result) const
Computes the column-wise (L^1) norm of this matrix.
gko::matrix::Dense::get_const_values
const value_type * get_const_values() const noexcept
Returns a pointer to the array of values of the matrix.
Definition: dense.hpp:883
gko::make_array_view
array< ValueType > make_array_view(std::shared_ptr< const Executor > exec, size_type size, ValueType *data)
Helper function to create an array view deducing the value type.
Definition: array.hpp:800
gko::matrix::Dense::extract_diagonal
std::unique_ptr< Diagonal< ValueType > > extract_diagonal() const override
Extracts the diagonal entries of the matrix into a vector.
gko::next_precision
typename detail::find_precision_impl< T, step >::type next_precision
Obtains the next move type of T in the singly-linked precision corresponding bfloat16/half.
Definition: math.hpp:465
gko::matrix::Dense::compute_absolute_inplace
void compute_absolute_inplace() override
Compute absolute inplace on each element.
gko::matrix::Dense::column_permute
std::unique_ptr< LinOp > column_permute(const array< int32 > *permutation_indices) const override
Returns a LinOp representing the column permutation of the Permutable object.
gko::EnableCloneable::convert_to
void convert_to(result_type *result) const override
Converts the implementer to an object of type result_type.
Definition: polymorphic_object.hpp:411
gko::previous_precision
typename detail::find_precision_impl< T, -step >::type previous_precision
Obtains the previous move type of T in the singly-linked precision corresponding bfloat16/half.
Definition: math.hpp:472
gko::matrix::Dense::create_const_view_of
static std::unique_ptr< const Dense > create_const_view_of(ptr_param< const Dense > other)
Creates a immutable Dense matrix, where the underlying array is a view of another Dense matrix' array...
Definition: dense.hpp:283
gko::matrix::Dense::at
ValueType & at(size_type idx) noexcept
Returns a single element of the matrix.
Definition: dense.hpp:946
gko::matrix::Dense::conj_transpose
std::unique_ptr< LinOp > conj_transpose() const override
Returns a LinOp representing the conjugate transpose of the Transposable object.
gko::int64
std::int64_t int64
64-bit signed integral type.
Definition: types.hpp:124
gko::matrix::Dense::compute_conj_dot
void compute_conj_dot(ptr_param< const LinOp > b, ptr_param< LinOp > result) const
Computes the column-wise dot product of conj(this matrix) and b.
gko::matrix::Ell
ELL is a matrix format where stride with explicit zeros is used such that all rows have the same numb...
Definition: csr.hpp:32
gko::ConvertibleTo
ConvertibleTo interface is used to mark that the implementer can be converted to the object of Result...
Definition: polymorphic_object.hpp:147
gko::matrix::Dense::create_submatrix
std::unique_ptr< Dense > create_submatrix(const span &rows, const span &columns)
Create a submatrix from the original matrix.
Definition: dense.hpp:1165
gko::int32
std::int32_t int32
32-bit signed integral type.
Definition: types.hpp:118
gko::matrix::Dense::compute_absolute
std::unique_ptr< absolute_type > compute_absolute() const override
Gets the AbsoluteLinOp.
gko::make_dense_view
std::unique_ptr< matrix::Dense< typename detail::pointee< VecPtr >::value_type > > make_dense_view(VecPtr &&vector)
Creates a view of a given Dense vector.
Definition: dense.hpp:1567
gko::Executor
The first step in using the Ginkgo library consists of creating an executor.
Definition: executor.hpp:615
gko::matrix::Hybrid
HYBRID is a matrix format which splits the matrix into ELLPACK and COO format.
Definition: coo.hpp:32
gko::array::get_const_data
const value_type * get_const_data() const noexcept
Returns a constant pointer to the block of memory used to store the elements of the array.
Definition: array.hpp:695
gko::matrix::permute_mode
permute_mode
Specifies how a permutation will be applied to a matrix.
Definition: permutation.hpp:42
gko::matrix::Dense::Dense
Dense(const Dense &)
Copy-constructs a Dense matrix.
gko::matrix::Sellp
SELL-P is a matrix format similar to ELL format.
Definition: csr.hpp:38
gko::make_const_dense_view
std::unique_ptr< const matrix::Dense< typename detail::pointee< VecPtr >::value_type > > make_const_dense_view(VecPtr &&vector)
Creates a view of a given Dense vector.
Definition: dense.hpp:1584
gko::EnableCloneable::move_to
void move_to(result_type *result) override
Converts the implementer to an object of type result_type by moving data from this object.
Definition: polymorphic_object.hpp:413
gko::PolymorphicObject::get_executor
std::shared_ptr< const Executor > get_executor() const noexcept
Returns the Executor of the object.
Definition: polymorphic_object.hpp:69
gko::array::get_size
size_type get_size() const noexcept
Returns the number of elements in the array.
Definition: array.hpp:669
gko::LinOp::get_size
const dim< 2 > & get_size() const noexcept
Returns the size of the operator.
Definition: lin_op.hpp:169
gko::local_span
A span that is used exclusively for local numbering.
Definition: range.hpp:137
gko::matrix::Dense::create_submatrix
std::unique_ptr< Dense > create_submatrix(const local_span &rows, const local_span &columns, dim< 2 > size)
Create a submatrix from the original matrix.
Definition: dense.hpp:1180
gko::matrix::Dense::create_real_view
std::unique_ptr< real_type > create_real_view()
Create a real view of the (potentially) complex original matrix.
gko::LinOp::LinOp
LinOp(const LinOp &)=default
Copy-constructs a LinOp.
gko::to_complex
typename detail::to_complex_s< T >::type to_complex
Obtain the type which adds the complex of complex/scalar type or the template parameter of class by a...
Definition: math.hpp:282
gko::matrix::Coo
COO stores a matrix in the coordinate matrix format.
Definition: coo.hpp:49