The simple solver with logging example.

This example depends on simple-solver, minimal-cuda-solver.

Table of contents
Introduction About the example The commented program	Results Comments about programming and debugging The plain program

Introduction

About the example

The commented program

#include <ginkgo/ginkgo.hpp>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <string>
namespace {
void print_vector(const std::string &name, const gko::matrix::Dense<> *vec)
{
    std::cout << name << " = [" << std::endl;
    for (int i = 0; i < vec->get_size()[0]; ++i) {
        std::cout << "    " << vec->at(i, 0) << std::endl;
    }
    std::cout << "];" << std::endl;
}
}  // namespace
int main(int argc, char *argv[])
{

Some shortcuts

using vec = gko::matrix::Dense<>;
using mtx = gko::matrix::Csr<>;
using cg = gko::solver::Cg<>;

Print version information

std::cout << gko::version_info::get() << std::endl;

Figure out where to run the code

std::shared_ptr<gko::Executor> exec;
if (argc == 1 || std::string(argv[1]) == "reference") {
    exec = gko::ReferenceExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "omp") {
    exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
           gko::CudaExecutor::get_num_devices() > 0) {
    exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
} else {
    std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
    std::exit(-1);
}

Read data

auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);

Let's declare a logger which prints to std::cout instead of printing to a file. We log all events except for all linop factory and polymorphic object events. Events masks are group of events which are provided for convenience.

std::shared_ptr<gko::log::Stream<>> stream_logger =
    gko::log::Stream<>::create(
        exec,
        gko::log::Logger::all_events_mask ^
            gko::log::Logger::linop_factory_events_mask ^
            gko::log::Logger::polymorphic_object_events_mask,
        std::cout);

Add stream_logger to the executor

exec->add_logger(stream_logger);

Add stream_logger only to the ResidualNormReduction criterion Factory Note that the logger will get automatically propagated to every criterion generated from this factory.

using ResidualCriterionFactory =
    gko::stop::ResidualNormReduction<>::Factory;
std::shared_ptr<ResidualCriterionFactory> residual_criterion =
    ResidualCriterionFactory::create().with_reduction_factor(1e-20).on(
        exec);
residual_criterion->add_logger(stream_logger);

Generate solver

auto solver_gen =
    cg::build()
        .with_criteria(
            residual_criterion,
            gko::stop::Iteration::build().with_max_iters(20u).on(exec))
        .on(exec);
auto solver = solver_gen->generate(A);

First we add facilities to only print to a file. It's possible to select events, using masks, e.g. only iterations mask: gko::log::Logger::iteration_complete_mask. See the documentation of Logger class for more information.

std::ofstream filestream("my_file.txt");
solver->add_logger(gko::log::Stream<>::create(
    exec, gko::log::Logger::all_events_mask, filestream));
solver->add_logger(stream_logger);

Add another logger which puts all the data in an object, we can later retrieve this object in our code. Here we only have want Executor and criterion check completed events.

std::shared_ptr<gko::log::Record> record_logger = gko::log::Record::create(
    exec, gko::log::Logger::executor_events_mask |
              gko::log::Logger::criterion_check_completed_mask);
exec->add_logger(record_logger);
residual_criterion->add_logger(record_logger);

Solve system

solver->apply(lend(b), lend(x));

Finally, get some data from record_logger and print the last memory location copied

auto &last_copy = record_logger->get().copy_completed.back();
std::cout << "Last memory copied was of size " << std::hex
          << std::get<0>(*last_copy).num_bytes << " FROM executor "
          << std::get<0>(*last_copy).exec << " pointer "
          << std::get<0>(*last_copy).location << " TO executor "
          << std::get<1>(*last_copy).exec << " pointer "
          << std::get<1>(*last_copy).location << std::dec << std::endl;

Also print the residual of the last criterion check event (where convergence happened)

auto residual =
    record_logger->get().criterion_check_completed.back()->residual.get();
auto residual_d = gko::as<gko::matrix::Dense<>>(residual);
print_vector("Residual", residual_d);

Print solution

std::cout << "Solution (x): \n";

write(std::cout, lend(x));

Calculate residual

    auto one = gko::initialize<vec>({1.0}, exec);
    auto neg_one = gko::initialize<vec>({-1.0}, exec);
    auto res = gko::initialize<vec>({0.0}, exec);
    A->apply(lend(one), lend(x), lend(neg_one), lend(b));
    b->compute_norm2(lend(res));
    std::cout << "Residual norm sqrt(r^T r): \n";
    write(std::cout, lend(res));
}

Results

This is the expected output:

[LOG] >>> apply started on A LinOp[gko::solver::Cg<double>,0x55ae09d923f0] with b LinOp[gko::matrix::Dense<double>,0x55ae09d928b0] and x LinOp[gko::matrix::Dense<double>,0x55ae09d92f10]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d91750] with Bytes[8]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d977e0] with Bytes[8]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[152]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97b40] with Bytes[152]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[152]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97d10] with Bytes[152]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[152]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97ee0] with Bytes[152]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[152]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d980b0] with Bytes[152]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98090] with Bytes[8]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98260] with Bytes[8]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d983b0] with Bytes[8]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98500] with Bytes[8]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[1]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98650] with Bytes[1]
[LOG] >>> Operation[gko::solver::cg::initialize_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko:
:Array<gko::stopping_status>*>,0x7ffcab765d60] started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::solver::cg::initialize_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko:
:Array<gko::stopping_status>*>,0x7ffcab765d60] completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab765980] started on Executor[gk
o::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab765980] completed on Executor[
gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[2]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98690] with Bytes[2]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d99310] with Bytes[8]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d99350] with Bytes[8]
[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab7657c0] started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab7657c0] completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] to Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] from Location[0x55ae09d97b40] to Location[0x55ae09d97d10] with Bytes[152]
[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] to Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] from Location[0x55ae09d97b40] to Location[0x55ae09d97d10] with Bytes[152]
[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab7659a0] started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab7659a0] completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> iteration 0 completed with solver LinOp[gko::solver::Cg<double>,0x55ae09d923f0] with residual LinOp[gko::matrix::Dense<double>,0x55ae09d930c0], solution LinOp[gko::matrix::Dense<double>,0x55ae09d92f10] and residual_norm LinOp[gko::LinOp const*,0]
[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x55ae09d99260] at iteration 0 with ID 1 and finalized set to 1
[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab765740] started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab765740] completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::stop::residual_norm_reduction::residual_norm_reduction_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffcab765980]
 started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::stop::residual_norm_reduction::residual_norm_reduction_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffcab765980]
 completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x55ae09d99260] at iteration 0 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[152]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d99820] with Bytes[152]
.
.
.
.
.
.
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98690]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98690]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d99350]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d99350]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d99310]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d99310]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98650]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98650]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98500]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98500]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d983b0]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d983b0]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98260]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98260]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98090]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d98090]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d980b0]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d980b0]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97ee0]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97ee0]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97d10]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97d10]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97b40]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97b40]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d977e0]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d977e0]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d91750]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d91750]
[LOG] >>> apply completed on A LinOp[gko::solver::Cg<double>,0x55ae09d923f0] with b LinOp[gko::matrix::Dense<double>,0x55ae09d928b0] and x LinOp[gko::matrix::Dense<double>,0x55ae09d92f10]
Last memory copied was of size 98 FROM executor 0x55ae09d8f2a0 pointer 55ae09d931d0 TO executor 0x55ae09d8f2a0 pointer 55ae09d998c0
Residual = [
    1.3067e-18
    -1.34263e-18
    -2.7754e-19
    2.35392e-20
    -2.25114e-19
    -1.35474e-20
    -1.82049e-19
    -2.48092e-19
    -4.57754e-19
    -1.28163e-18
    -1.04918e-18
    -5.88231e-19
    -8.463e-19
    -2.87785e-18
    -4.06072e-18
    -9.40979e-18
    -1.07071e-17
    -4.14666e-17
    -2.75923e-17
];
Solution (x):
%%MatrixMarket matrix array real general
19 1
0.252218
0.108645
0.0662811
0.0630433
0.0384088
0.0396536
0.0402648
0.0338935
0.0193098
0.0234653
0.0211499
0.0196413
0.0199151
0.0181674
0.0162722
0.0150714
0.0107016
0.0121141
0.0123025
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d9a310] with Bytes[8]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97ec0] with Bytes[8]
[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] with Bytes[8]
[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d99370] with Bytes[8]
[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab765d50] started on Executor[gk
o::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab765d50] completed on Executor[
gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab765dc0] started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffcab765dc0] completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0]
Residual norm sqrt(r^T r):
%%MatrixMarket matrix array real general
1 1
8.87107e-16
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d99370]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d99370]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97ec0]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d97ec0]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d9a310]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d9a310]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d931d0]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d931d0]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d93020]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d93020]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d92830]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d92830]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d925b0]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d925b0]
[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d93360]
[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x55ae09d8f2a0] at Location[0x55ae09d93360]

Comments about programming and debugging

The plain program

/*******************************<GINKGO LICENSE>******************************
Copyright (c) 2017-2019, the Ginkgo authors
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
******************************<GINKGO LICENSE>*******************************/
#include <ginkgo/ginkgo.hpp>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <string>
namespace {
void print_vector(const std::string &name, const gko::matrix::Dense<> *vec)
{
    std::cout << name << " = [" << std::endl;
    for (int i = 0; i < vec->get_size()[0]; ++i) {
        std::cout << "    " << vec->at(i, 0) << std::endl;
    }
    std::cout << "];" << std::endl;
}
}  // namespace
int main(int argc, char *argv[])
{
    using vec = gko::matrix::Dense<>;
    using mtx = gko::matrix::Csr<>;
    using cg = gko::solver::Cg<>;
    std::cout << gko::version_info::get() << std::endl;
    std::shared_ptr<gko::Executor> exec;
    if (argc == 1 || std::string(argv[1]) == "reference") {
        exec = gko::ReferenceExecutor::create();
    } else if (argc == 2 && std::string(argv[1]) == "omp") {
        exec = gko::OmpExecutor::create();
    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
               gko::CudaExecutor::get_num_devices() > 0) {
        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
    } else {
        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
        std::exit(-1);
    }
    auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
    auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
    auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
    std::shared_ptr<gko::log::Stream<>> stream_logger =
        gko::log::Stream<>::create(
            exec,
            gko::log::Logger::all_events_mask ^
                gko::log::Logger::linop_factory_events_mask ^
                gko::log::Logger::polymorphic_object_events_mask,
            std::cout);
    exec->add_logger(stream_logger);
    using ResidualCriterionFactory =
        gko::stop::ResidualNormReduction<>::Factory;
    std::shared_ptr<ResidualCriterionFactory> residual_criterion =
        ResidualCriterionFactory::create().with_reduction_factor(1e-20).on(
            exec);
    residual_criterion->add_logger(stream_logger);
    auto solver_gen =
        cg::build()
            .with_criteria(
                residual_criterion,
                gko::stop::Iteration::build().with_max_iters(20u).on(exec))
            .on(exec);
    auto solver = solver_gen->generate(A);
    std::ofstream filestream("my_file.txt");
    solver->add_logger(gko::log::Stream<>::create(
        exec, gko::log::Logger::all_events_mask, filestream));
    solver->add_logger(stream_logger);
    std::shared_ptr<gko::log::Record> record_logger = gko::log::Record::create(
        exec, gko::log::Logger::executor_events_mask |
                  gko::log::Logger::criterion_check_completed_mask);
    exec->add_logger(record_logger);
    residual_criterion->add_logger(record_logger);
    solver->apply(lend(b), lend(x));
    auto &last_copy = record_logger->get().copy_completed.back();
    std::cout << "Last memory copied was of size " << std::hex
              << std::get<0>(*last_copy).num_bytes << " FROM executor "
              << std::get<0>(*last_copy).exec << " pointer "
              << std::get<0>(*last_copy).location << " TO executor "
              << std::get<1>(*last_copy).exec << " pointer "
              << std::get<1>(*last_copy).location << std::dec << std::endl;
    auto residual =
        record_logger->get().criterion_check_completed.back()->residual.get();
    auto residual_d = gko::as<gko::matrix::Dense<>>(residual);
    print_vector("Residual", residual_d);
    std::cout << "Solution (x): \n";
    write(std::cout, lend(x));
    auto one = gko::initialize<vec>({1.0}, exec);
    auto neg_one = gko::initialize<vec>({-1.0}, exec);
    auto res = gko::initialize<vec>({0.0}, exec);
    A->apply(lend(one), lend(x), lend(neg_one), lend(b));
    b->compute_norm2(lend(res));
    std::cout << "Residual norm sqrt(r^T r): \n";
    write(std::cout, lend(res));
}