Add detailed time measurements for assembly.

Author: TomFischer

NumLib/ODESolver/NonlinearSolver.cpp

@@ -24,6 +24,10 @@
 #include "NumLib/Exceptions.h"
 #include "PETScNonlinearSolver.h"
 
+double time_pure_assembly {};
+double time_insert_in_matrix {};
+double time_dof_table_lookup {};
+
 namespace NumLib
 {
 namespace detail
@@ -165,9 +169,15 @@ NonlinearSolverStatus NonlinearSolver<NonlinearSolverTag::Picard>::solve(
 
         sys.preIteration(iteration, x_new_process);
 
+        // reset the time variables
+        time_pure_assembly = 0;
+        time_insert_in_matrix = 0;
+        time_dof_table_lookup = 0;
+
         BaseLib::RunTime time_assembly;
         time_assembly.start();
         sys.assemble(x_new, x_prev, process_id);
+        double asm_time_elapsed = time_assembly.elapsed();
         sys.getA(A);
         sys.getRhs(*x_prev[process_id], rhs);
 
@@ -186,6 +196,12 @@ NonlinearSolverStatus NonlinearSolver<NonlinearSolverTag::Picard>::solve(
         }
 
         INFO("[time] Assembly took {:g} s.", time_assembly.elapsed());
+        INFO(
+            "[time] Pure assembly took {:g} s, insert in matrix took {:g} s, "
+            "dof table lookup took {:g} s.",
+            time_pure_assembly, time_insert_in_matrix, time_dof_table_lookup);
+        INFO("[time] Assembly took {:g} s, getA + getRhs took {:g} s.",
+             asm_time_elapsed, time_assembly.elapsed() - asm_time_elapsed);
 
         // Subtract non-equilibrium initial residuum if set
         if (_r_neq != nullptr)
@@ -371,6 +387,11 @@ NonlinearSolverStatus NonlinearSolver<NonlinearSolverTag::Newton>::solve(
 
         sys.preIteration(iteration, *x[process_id]);
 
+        // reset the time variables
+        time_pure_assembly = 0;
+        time_insert_in_matrix = 0;
+        time_dof_table_lookup = 0;
+
         BaseLib::RunTime time_assembly;
         time_assembly.start();
         bool mpi_rank_assembly_ok = true;
@@ -390,9 +411,13 @@ NonlinearSolverStatus NonlinearSolver<NonlinearSolverTag::Newton>::solve(
         {
             break;
         }
+        INFO(
+            "[time] Pure assembly took {:g} s, insert in matrix took {:g} s, "
+            "dof table lookup took {:g} s.",
+            time_pure_assembly, time_insert_in_matrix, time_dof_table_lookup);
+        INFO("[time] Assembly took {:g} s.", time_assembly.elapsed());
         sys.getResidual(*x[process_id], *x_prev[process_id], res);
         sys.getJacobian(J);
-        INFO("[time] Assembly took {:g} s.", time_assembly.elapsed());
 
         // Subtract non-equilibrium initial residuum if set
         if (_r_neq != nullptr)

NumLib/ODESolver/NonlinearSolver.h

@@ -19,6 +19,10 @@
 #include "NonlinearSystem.h"
 #include "Types.h"
 
+extern double time_pure_assembly;
+extern double time_insert_in_matrix;
+extern double time_dof_table_lookup;
+
 namespace BaseLib
 {
 class ConfigTree;

ProcessLib/VectorMatrixAssembler.cpp

@@ -11,6 +11,15 @@
 #include "VectorMatrixAssembler.h"
 
 #include <cassert>
+<<<<<<< HEAD
+=======
+#include <functional>  // for std::reference_wrapper.
+
+#include "BaseLib/RunTime.h"
+#include "NumLib/DOF/DOFTableUtil.h"
+#include "MathLib/LinAlg/Eigen/EigenMapTools.h"
+#include "LocalAssemblerInterface.h"
+>>>>>>> 82be5ea4a3 (Add detailed time measurements for assembly.)
 
 #include "CoupledSolutionsForStaggeredScheme.h"
 #include "LocalAssemblerInterface.h"
@@ -43,6 +52,8 @@ void VectorMatrixAssembler::assemble(
     std::vector<GlobalVector*> const& x_prev, int const process_id,
     GlobalMatrix* M, GlobalMatrix* K, GlobalVector* b)
 {
+    BaseLib::RunTime timer;
+    timer.start();
     std::vector<std::vector<GlobalIndexType>> indices_of_processes;
     indices_of_processes.reserve(dof_tables.size());
     transform(cbegin(dof_tables), cend(dof_tables),
@@ -51,12 +62,15 @@ void VectorMatrixAssembler::assemble(
               { return NumLib::getIndices(mesh_item_id, *dof_table); });
 
     auto const& indices = indices_of_processes[process_id];
+    time_dof_table_lookup += timer.elapsed();
+
     _local_M_data.clear();
     _local_K_data.clear();
     _local_b_data.clear();
 
     std::size_t const number_of_processes = x.size();
     // Monolithic scheme
+    timer.start();
     if (number_of_processes == 1)
     {
         auto const local_x = x[process_id]->get(indices);
@@ -78,11 +92,15 @@ void VectorMatrixAssembler::assemble(
             t, dt, local_x, local_x_prev, process_id, _local_M_data,
             _local_K_data, _local_b_data);
     }
+    time_pure_assembly += timer.elapsed();
 
+    timer.start();
     auto const num_r_c = indices.size();
     auto const r_c_indices =
         NumLib::LocalToGlobalIndexMap::RowColumnIndices(indices, indices);
+    time_dof_table_lookup += timer.elapsed();
 
+    timer.start();
     if (M && !_local_M_data.empty())
     {
         auto const local_M = MathLib::toMatrix(_local_M_data, num_r_c, num_r_c);
@@ -99,6 +117,7 @@ void VectorMatrixAssembler::assemble(
         b->add(indices, _local_b_data);
     }
 
+    time_insert_in_matrix += timer.elapsed();
     _local_output(t, process_id, mesh_item_id, _local_M_data, _local_K_data,
                   _local_b_data);
 }