/// <summary>
/// Solves a symmetric, definite system using the conjugate gradient algorithm.
/// </summary>
/// <param name="Matrix">the matrix of the linear problem.</param>
/// <param name="X">Output, (hopefully) the solution.</param>
/// <param name="B">Input, the right-hand-side.</param>
/// <param name="MaxIterations"></param>
/// <param name="Tolerance"></param>
/// <returns>Actual number of iterations.</returns>
static public int Solve_CG <V, W>(this IMutableMatrixEx Matrix, V X, W B, int MaxIterations = 100000, double Tolerance = 1.0e-10)
where V : IList <double>
where W : IList <double> //
{
using (var slv = new ilPSP.LinSolvers.monkey.CG()) {
slv.MaxIterations = MaxIterations;
slv.Tolerance = Tolerance;
slv.DevType = monkey.DeviceType.CPU;
slv.DefineMatrix(Matrix);
var SolRes = slv.Solve(X, B.ToArray());
return(SolRes.NoOfIterations);
}
}
protected override double RunSolverOneStep(int TimestepNo, double phystime, double dt)
{
using (new FuncTrace()) {
base.NoOfTimesteps = -1;
base.TerminationKey = true;
int D = this.GridData.SpatialDimension;
this.Err.Clear();
this.Err.Acc(-1.0, this.U);
// compare linear vs. nonlinear evaluation
// =======================================
LinearNonlinComparisonTest();
// call solver
// ===========
if (this.mode == TestMode.Solve)
{
var solver = new ilPSP.LinSolvers.monkey.CG();
solver.MaxIterations = 20000;
solver.Tolerance = 1.0e-12;
solver.DevType = ilPSP.LinSolvers.monkey.DeviceType.CPU;
solver.DefineMatrix(this.OperatorMtx);
double[] _rhs = this.RHS.CoordinateVector.ToArray();
_rhs.AccV(-1.0, this.bnd.CoordinateVector);
var solRes = solver.Solve(this.U.CoordinateVector, _rhs);
Console.WriteLine("sparse solver result: " + solRes.ToString());
Assert.IsTrue(solRes.Converged);
}
else if (this.mode == TestMode.CheckResidual)
{
// residual computation is done anyway...
}
else
{
throw new NotImplementedException();
}
// Residual computation
// ====================
{
this.Residual.Clear();
this.Residual.Acc(1.0, this.bnd);
this.OperatorMtx.SpMVpara(1.0, this.U.CoordinateVector, 1.0, this.Residual.CoordinateVector);
this.Residual.Acc(-1.0, this.RHS);
L2ResidualNorm = D.ForLoop(delegate(int i) {
var f = this.Residual[i];
double L2Norm = f.L2Norm();
Console.WriteLine("L2 " + f.Identification + ": " + L2Norm);
return(L2Norm);
});
}
// Error computation
// =================
{
this.Err.Acc(1.0, this.U);
L2ErrorNorm = D.ForLoop(delegate(int i) {
var f = this.Err[i];
double L2Norm = f.L2Norm();
Console.WriteLine("L2 " + f.Identification + ": " + L2Norm);
return(L2Norm);
});
}
return(0.0);
}
}
/// <summary>
/// projects some DG field onto this
/// </summary>
/// <param name="alpha"></param>
/// <param name="DGField"></param>
/// <param name="_cm">optional restriction to computational domain</param>
/// <remarks>
/// This method computes an exact
/// L2-projection of the DG-field onto the SpecFEM-space, so a global linear system, which contains all
/// DOF's, has to be solved.
/// In contrast, <see cref="ProjectDGFieldCheaply"/> performs an approximate projection which only involves
/// local operations for each cell.
/// </remarks>
public void ProjectDGField(double alpha, ConventionalDGField DGField, CellMask _cm = null)
{
using (var trx = new Transceiver(this.Basis)) {
CellMask cm = _cm;
if (cm == null)
{
cm = CellMask.GetFullMask(this.Basis.GridDat);
}
int J = m_Basis.GridDat.Cells.NoOfLocalUpdatedCells;
var Trafo = m_Basis.GridDat.ChefBasis.Scaling;
var C2N = m_Basis.CellNode_To_Node;
var MtxM2N = m_Basis.m_Modal2Nodal;
var CellData = this.Basis.GridDat.Cells;
// compute RHS
// ===========
var b = MultidimensionalArray.Create(this.m_Basis.NoOfLocalNodes);
{
int[] _K = m_Basis.NodesPerCell;
int L = m_Basis.ContainingDGBasis.Length;
double[][] _NodalCoordinates = _K.Select(K => new double[K]).ToArray(); // temporary storage for nodal coordinates per cell
// 1st idx: ref. elm., 2nd idx: node index
double[] ModalCoordinates = new double[L];
foreach (Chunk cnk in cm)
{
int j0 = cnk.i0;
int jE = cnk.JE;
for (int j = j0; j < jE; j++) // loop over cells...
{
int iKref = CellData.GetRefElementIndex(j);
double[] NodalCoordinates = _NodalCoordinates[iKref];
int K = _K[iKref];
if (!CellData.IsCellAffineLinear(j))
{
throw new NotSupportedException();
}
// Get DG coordinates
Array.Clear(ModalCoordinates, 0, L);
int Lmin = Math.Min(L, DGField.Basis.GetLength(j));
for (int l = 0; l < Lmin; l++)
{
ModalCoordinates[l] = DGField.Coordinates[j, l];
}
var tr = 1.0 / Trafo[j];
// transform
//DGField.Coordinates.GetRow(j, ModalCoordinates);
ModalCoordinates.ClearEntries();
for (int l = 0; l < Lmin; l++)
{
ModalCoordinates[l] = DGField.Coordinates[j, l];
}
MtxM2N[iKref].GEMV(tr, ModalCoordinates, 0.0, NodalCoordinates, transpose: true);
// collect coordinates for cell 'j':
for (int k = 0; k < K; k++)
{
int _c2n = C2N[j, k];
b[_c2n] += NodalCoordinates[k];
}
}
}
}
trx.AccumulateGather(b);
/*
*
* var bcheck = new double[b.Length];
* {
* var polys = this.Basis.NodalBasis;
*
*
* CellQuadrature.GetQuadrature(new int[] { K },
* this.Basis.GridDat.Context,
* (new CellQuadratureScheme()).Compile(this.Basis.GridDat, this.Basis.ContainingDGBasis.Degree*2),
* delegate(MultidimensionalArray NodesUntransformed) { // Del_CreateNodeSetFamily
* var NSC = this.Basis.GridDat.Context.NSC;
* return new NodeSetController.NodeSetContainer[] { NSC.CreateContainer(NodesUntransformed) };
* },
* delegate(int i0, int Length, int _NoOfNodes, MultidimensionalArray EvalResult) {
* var PolyAtNode = MultidimensionalArray.Create(K, _NoOfNodes);
* for (int k = 0; k < K; k++) {
* polys[k].Evaluate(PolyAtNode.ExtractSubArrayShallow(k, -1), this.Basis.GridDat.Context.NSC.Current_NodeSetFamily[0].NodeSet);
* }
*
* var DGFatNodes = MultidimensionalArray.Create(Length, _NoOfNodes);
* DGField.Evaluate(i0, Length, 0, DGFatNodes);
*
* //for(int i = 0; i < Length; i++) {
* // for (int n = 0; n < _NoOfNodes; n++) {
* // for (int k = 0; k < K; k++) {
* // for (int l = 0; l < K; l++) {
* // EvalResult[i, n, k, l] = PolyAtNode[k, n]*PolyAtNode[l, n];
* // }
* // }
* // }
* //}
*
* EvalResult.Multiply(1.0, PolyAtNode, DGFatNodes, 0.0, "jnk", "kn", "jn");
*
* //double errSum = 0;
* //for (int i = 0; i < Length; i++) {
* // for (int n = 0; n < _NoOfNodes; n++) {
* // for (int k = 0; k < K; k++) {
* // for (int l = 0; l < K; l++) {
* // double soll = PolyAtNode[k, n]*PolyAtNode[l, n];
* // errSum += Math.Abs(soll - EvalResult[i, n, k, l]);
* // }
* // }
* // }
* //}
* //Console.WriteLine("errsum = " + errSum);
* },
* delegate(int i0, int Length, MultidimensionalArray ResultsOfIntegration) { // SaveIntegrationResults
* for (int i = 0; i < Length; i++) {
* int jCell = i + i0;
*
* for (int k = 0; k < K; k++) {
* bcheck[C2N[jCell, k]] += ResultsOfIntegration[i, k];
* }
*
* //CellMass[jCell] = new FullMatrix(K, K);
* //CellMass[jCell].Initialize(ResultsOfIntegration.ExtractSubArrayShallow(i, -1, -1));
* }
* }).Execute();
*
*
* double f**k = GenericBlas.L2Dist(b, bcheck);
* Console.WriteLine("Distance error = " + f**k);
*
* }
*
*
*/
if (_cm == null)
{
// full domain projection branch
// +++++++++++++++++++++++++++++
var x = new double[this.Basis.NoOfLocalOwnedNodes];
var solStat = m_Basis.MassSolver.Solve(x, b.ExtractSubArrayShallow(new int[] { 0 }, new int[] { this.Basis.NoOfLocalOwnedNodes - 1 }).To1DArray());
{
if (solStat.Converged == false)
{
throw new ArithmeticException("DG -> SpecFEM Projection failed because the Mass matrix solver did not converge.");
}
double[] chk = b.ExtractSubArrayShallow(new int[] { 0 }, new int[] { this.Basis.NoOfLocalOwnedNodes - 1 }).To1DArray();
this.Basis.MassMatrix.SpMVpara(-1.0, x, 1.0, chk);
double chk_nomr = chk.L2Norm();
if (chk_nomr >= 1.0e-8)
{
throw new ArithmeticException(string.Format("DG -> SpecFEM Projection failed: solver converged, but with high residual {0}.", chk_nomr.ToStringDot()));
}
}
//m_Basis.MassMatrix.SpMV(1.0, b, 0.0, x);
m_Coordinates.ExtractSubArrayShallow(new int[] { 0 }, new int[] { this.Basis.NoOfLocalOwnedNodes - 1 }).AccVector(alpha, x);
//m_Coordinates.AccV(alpha, b);
}
else
{
// restricted domain projection branch
// +++++++++++++++++++++++++++++++++++
List <int> OccupiedRows_Global = new List <int>();
//List<int> OccupiedRows_Local = new List<int>();
var MM = Basis.ComputeMassMatrix(cm);
int i0 = MM.RowPartitioning.i0, iE = MM.RowPartitioning.iE;
for (int i = i0; i < iE; i++)
{
if (MM.GetNoOfNonZerosPerRow(i) > 0)
{
OccupiedRows_Global.Add(i);
//OccupiedRows_Local.Add(i - i0);
}
}
var CompressedPart = new Partitioning(OccupiedRows_Global.Count);
var CompressedMM = new MsrMatrix(CompressedPart);
MM.WriteSubMatrixTo(CompressedMM, OccupiedRows_Global, default(int[]), OccupiedRows_Global, default(int[]));
var b_sub = new double[OccupiedRows_Global.Count];
//try {
b_sub.AccV(1.0, b.To1DArray(), default(int[]), OccupiedRows_Global, b_index_shift: -i0);
//} catch(Exception e) {
// Debugger.Launch();
//}
//csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD);
var x_sub = new double[b_sub.Length];
var solver = new ilPSP.LinSolvers.monkey.CG();
solver.MatrixType = ilPSP.LinSolvers.monkey.MatrixType.CCBCSR;
solver.DevType = ilPSP.LinSolvers.monkey.DeviceType.CPU;
solver.ConvergenceType = ConvergenceTypes.Absolute;
solver.Tolerance = 1.0e-12;
solver.DefineMatrix(CompressedMM);
var solStat = solver.Solve(x_sub, b_sub.CloneAs());
{
if (solStat.Converged == false)
{
throw new ArithmeticException("DG -> SpecFEM Projection failed because the Mass matrix solver did not converge.");
}
var chk = b_sub;
CompressedMM.SpMVpara(-1.0, x_sub, 1.0, chk);
double chk_nomr = chk.L2Norm();
if (chk_nomr >= 1.0e-8)
{
throw new ArithmeticException(string.Format("DG -> SpecFEM Projection failed: solver converged, but with high residual {0}.", chk_nomr.ToStringDot()));
}
}
double[] x = new double[this.Basis.NoOfLocalOwnedNodes];
x.AccV(1.0, x_sub, OccupiedRows_Global, default(int[]), acc_index_shift: -i0);
m_Coordinates.ExtractSubArrayShallow(new int[] { 0 }, new int[] { this.Basis.NoOfLocalOwnedNodes - 1 }).AccVector(alpha, x);
}
trx.Scatter(this.m_Coordinates);
}
}
/// <summary>
/// Solution of the system
/// <see cref="LaplaceMtx"/>*<see cref="T"/> + <see cref="LaplaceAffine"/> = <see cref="RHS"/>
/// using a black-box solver
/// </summary>
private void ClassicSolve(out double mintime, out double maxtime, out bool Converged, out int NoOfIter)
{
mintime = double.MaxValue;
maxtime = double.MinValue;
Converged = false;
NoOfIter = int.MaxValue;
for (int i = 0; i < base.Control.NoOfSolverRuns; i++)
{
// create sparse solver
// --------------------
ISparseSolver ipSolver;
LinearSolverCode solvercodes = LinearSolverCode.classic_pardiso;
switch (solvercodes)
{
case LinearSolverCode.classic_pardiso:
ipSolver = new ilPSP.LinSolvers.PARDISO.PARDISOSolver()
{
CacheFactorization = true,
UseDoublePrecision = true
};
break;
case LinearSolverCode.classic_mumps:
ipSolver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
break;
case LinearSolverCode.classic_cg:
ipSolver = new ilPSP.LinSolvers.monkey.CG()
{
MaxIterations = 1000000,
Tolerance = 1.0e-10,
DevType = ilPSP.LinSolvers.monkey.DeviceType.Cuda
};
break;
default:
throw new ArgumentException();
}
ipSolver.DefineMatrix(LaplaceMtx);
// call solver
// -----------
T.Clear();
Console.WriteLine("RUN " + i + ": solving system...");
var RHSvec = RHS.CoordinateVector.ToArray();
BLAS.daxpy(RHSvec.Length, -1.0, this.LaplaceAffine, 1, RHSvec, 1);
T.Clear();
SolverResult solRes = ipSolver.Solve(T.CoordinateVector, RHSvec);
mintime = Math.Min(solRes.RunTime.TotalSeconds, mintime);
maxtime = Math.Max(solRes.RunTime.TotalSeconds, maxtime);
Converged = solRes.Converged;
NoOfIter = solRes.NoOfIterations;
Console.WriteLine("Pardiso phase 11: " + ilPSP.LinSolvers.PARDISO.PARDISOSolver.Phase_11.Elapsed.TotalSeconds);
Console.WriteLine("Pardiso phase 22: " + ilPSP.LinSolvers.PARDISO.PARDISOSolver.Phase_22.Elapsed.TotalSeconds);
Console.WriteLine("Pardiso phase 33: " + ilPSP.LinSolvers.PARDISO.PARDISOSolver.Phase_33.Elapsed.TotalSeconds);
ipSolver.Dispose();
}
}
