public void Init(MultigridOperator op)
{
var M = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_mgop = op;
if (!M.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!M.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
Mtx = M;
int L = M.RowPartitioning.LocalLength;
diag = new double[L];
int i0 = Mtx.RowPartitioning.i0;
for (int i = 0; i < L; i++)
{
diag[i] = Mtx[i0 + i, i0 + i];
}
}
/// <summary>
/// defines the problem matrix
/// </summary>
public void Init(MultigridOperator op)
{
// init & check
// ============
this.m_MgOperator = op;
//OpMatrix = op.OperatorMatrix;
OpMatrix = new ilPSP.LinSolvers.monkey.CPU.RefMatrix(op.OperatorMatrix.ToMsrMatrix());
var MgMap = op.Mapping;
if (!OpMatrix.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!OpMatrix.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
// init solver chain
// =================
if (SolverChain == null || SolverChain.Length <= 0)
{
throw new NotSupportedException("Illegal configuration.");
}
for (int i = 0; i < SolverChain.Length; i++)
{
SolverChain[i].Init(op);
}
}
/// <summary>
/// Updating the <see cref="CurrentLin"/> -- operator;
/// </summary>
/// <param name="CurrentState">linearization point</param>
protected void Update(IEnumerable <DGField> CurrentState)
{
if (!(this.ProblemMapping.BasisS.Count == CurrentState.Count()))
{
throw new ArgumentException("missmatch in number of fields.");
}
BlockMsrMatrix OpMtxRaw, MassMtxRaw;
double[] OpAffineRaw;
this.m_AssembleMatrix(out OpMtxRaw, out OpAffineRaw, out MassMtxRaw, CurrentState.ToArray());
CurrentLin = new MultigridOperator(this.m_AggBasisSeq, this.ProblemMapping,
OpMtxRaw.CloneAs(), MassMtxRaw,
this.m_MultigridOperatorConfig);
OpAffineRaw = OpAffineRaw.CloneAs();
if (this.RHSRaw != null)
{
OpAffineRaw.AccV(-1.0, this.RHSRaw);
}
if (LinearizationRHS == null || LinearizationRHS.Length != this.CurrentLin.Mapping.LocalLength)
{
LinearizationRHS = new double[this.CurrentLin.Mapping.LocalLength];
}
else
{
LinearizationRHS.ClearEntries();
}
CurrentLin.TransformRhsInto(OpAffineRaw, this.LinearizationRHS);
this.LinearizationRHS.ScaleV(-1.0);
}
ilPSP.LinSolvers.monkey.CPU.RefMatrix m_Matrix; // im Moment schneller, ca 5X
//BlockMsrMatrix m_Matrix;
public void Init(MultigridOperator op)
{
using (new FuncTrace()) {
var M = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_MgOp = op;
if (!M.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!M.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
//this.m_Matrix = M;
this.m_Matrix = new ilPSP.LinSolvers.monkey.CPU.RefMatrix(M.ToMsrMatrix());
/*
* int n = m_Matrix.RowPartitioning.LocalLength;
* if(n > 50000) {
* var _Matrix = new ilPSP.LinSolvers.monkey.CPU.RefMatrix(M.ToMsrMatrix());
*
* double[] xTest = new double[n];
* double[] bTest = new double[n];
* Random r = new Random(123);
* for(int i = 0; i < n; i++) {
* xTest[i] = r.NextDouble();
* bTest[i] = r.NextDouble();
* }
*
*
* double[] b1 = bTest.CloneAs();
* double[] x1 = bTest.CloneAs();
* Stopwatch monkey = new Stopwatch();
* monkey.Start();
* for (int i = 0; i < 100; i++)
* _Matrix.SpMV(1.0, x1, -0.1, b1);
* monkey.Stop();
*
* double[] b2 = bTest.CloneAs();
* double[] x2 = bTest.CloneAs();
* Stopwatch block = new Stopwatch();
* block.Start();
* for (int i = 0; i < 100; i++)
* m_Matrix.SpMV(1.0, x1, -0.1, b1);
* block.Stop();
*
* Console.WriteLine("SPMV monkey: " + monkey.Elapsed.TotalSeconds);
* Console.WriteLine("SPMV block MSR: " + block.Elapsed.TotalSeconds);
* }
*/
if (Precond != null)
{
Precond.Init(op);
}
}
}
public void Init(MultigridOperator op)
{
int D = op.GridData.SpatialDimension;
CodName = new string[] { "momX", "momY", "div", "constitutiveXX", "constitutiveXY", "constitutiveYY" };
Params = new string[] { "VelocityX_GradientX", "VelocityX_GradientY", "VelocityY_GradientX", "VelocityY_GradientY" };
DomName = new string[] { "VelocityX", "VelocityY", "Pressure", "StressXX", "StressXY", "StressYY" };
LocalOp = new SpatialOperator(DomName, Params, CodName, (A, B, C) => 4);
LocalOp.EquationComponents["constitutiveXX"].Add(new LocalJacobiFlux()
{
m_component = 0, We = m_We
});
LocalOp.EquationComponents["constitutiveXY"].Add(new LocalJacobiFlux()
{
m_component = 1, We = m_We
});
LocalOp.EquationComponents["constitutiveYY"].Add(new LocalJacobiFlux()
{
m_component = 2, We = m_We
});
LocalOp.Commit();
var U0 = ((BoSSS.Foundation.CoordinateMapping)op.Mapping.ProblemMapping).Fields.GetSubVector(0, 2);
Basis U0Basis = new Basis(op.Mapping.GridData, 0);
VectorField <SinglePhaseField> VelocityXGradient = new VectorField <SinglePhaseField>(D, U0Basis, "VelocityX_Gradient", SinglePhaseField.Factory);
VectorField <SinglePhaseField> VelocityYGradient = new VectorField <SinglePhaseField>(D, U0Basis, "VelocityY_Gradient", SinglePhaseField.Factory);
VelocityXGradient.Clear();
VelocityXGradient.GradientByFlux(1.0, U0[0]);
VelocityYGradient.Clear();
VelocityYGradient.GradientByFlux(1.0, U0[1]);
var Parameters = ArrayTools.Cat <DGField>(VelocityXGradient, VelocityYGradient);
LocalMatrix = LocalOp.ComputeMatrix(op.BaseGridProblemMapping, Parameters, op.BaseGridProblemMapping);
ConstEqIdx = op.Mapping.ProblemMapping.GetSubvectorIndices(true, 3, 4, 5);
P = (BlockMsrMatrix)op.MassMatrix.Clone();
for (int i = ConstEqIdx[0]; i <= ConstEqIdx.Length; i++)
{
for (int j = ConstEqIdx[0]; j <= ConstEqIdx.Length; j++)
{
if (LocalMatrix[i, j] != 0)
{
P[i, j] = LocalMatrix[i, j];
}
}
}
LocalMatrix.SaveToTextFileSparse("LocalMatrix");
op.MassMatrix.SaveToTextFileSparse("MassMatrix");
P.SaveToTextFileSparse("PrecondMatrix");
}
public IList <double[]> OrthonormalMultigridDecomposition(double[] Vec, bool decompose = true)
{
// vector length on level 0
int L0 = DecompositionOperator.Mapping.LocalLength;
if (Vec.Length != L0)
{
throw new ArgumentException("Mismatch in vector length.", "Vec");
}
List <double[]> OrthoVecs = new List <double[]>();
OrthoVecs.Add(Vec.CloneAs());
double l2pow2_Vec = OrthoVecs[0].L2NormPow2().MPISum();
MultigridOperator coarsest = null;
for (var mgop = this.DecompositionOperator.CoarserLevel; mgop != null; mgop = mgop.CoarserLevel)
{
int L = mgop.Mapping.LocalLength;
int iLevel = mgop.LevelIndex;
OrthoVecs.Add(new double[L]);
mgop.Restrict(OrthoVecs[iLevel - 1], OrthoVecs[iLevel]);
coarsest = mgop;
}
double Check_l2pow2_OrthoVecsTotal = 0.0;
for (var mgop = this.DecompositionOperator.CoarserLevel; mgop != null; mgop = mgop.CoarserLevel)
{
int iLevel = mgop.LevelIndex;
if (decompose)
{
mgop.Prolongate(-1.0, OrthoVecs[iLevel - 1], 1.0, OrthoVecs[iLevel]);
}
coarsest = mgop;
}
foreach (double[] OrthoVec in OrthoVecs)
{
Check_l2pow2_OrthoVecsTotal += OrthoVec.L2NormPow2().MPISum();
}
// if the vectors are really orthonormal, their squared L2-norms must be additive!
if (decompose && this.DecompositionOperator_IsOrthonormal)
{
Debug.Assert((Math.Abs(Check_l2pow2_OrthoVecsTotal - l2pow2_Vec) / l2pow2_Vec < 1.0e-8), "something wrong with orthonormal decomposition");
}
return(OrthoVecs);
}
/// <summary>
/// Returns the multigrid blocking.
/// </summary>
internal override IEnumerable <List <int> > GetBlocking(MultigridOperator op)
{
AggregationGrid thisLevel = op.Mapping.AggGrid;
List <AggregationGrid> blockLevelS = new List <AggregationGrid>();
blockLevelS.Add(thisLevel);
MultigridOperator blokOp = op;
for (int i = 0; i < this.Depth; i++)
{
if (blokOp.CoarserLevel == null)
{
throw new NotSupportedException("Not enough multigrid levels set to support a depth of " + m_Depht + ".");
}
blokOp = blokOp.CoarserLevel;
blockLevelS.Add(blokOp.Mapping.AggGrid);
}
AggregationGrid blckLevel = blockLevelS.Last(); // the cells of this level form the additive-Schwarz blocks
int NoBlocks = blckLevel.iLogicalCells.NoOfLocalUpdatedCells; // each cell of 'blckLevel' forms a block
List <int>[] Blocks = NoBlocks.ForLoop(l => new List <int>());
#if DEBUG
bool[] checkOnce = new bool[thisLevel.iLogicalCells.NoOfLocalUpdatedCells];
#endif
for (int iBlk = 0; iBlk < NoBlocks; iBlk++)
{
if (blockLevelS.Count == 0)
{
Blocks[iBlk].Add(iBlk); // the cell itself is the multigrid block (either Depth is 0, or no more MG level available).
}
else
{
int[] CoarseCell = blckLevel.jCellCoarse2jCellFine[iBlk];
CollectBlock(Blocks[iBlk], blockLevelS, 0, CoarseCell);
}
#if DEBUG
foreach (int j in Blocks[iBlk])
{
Debug.Assert(j >= 0);
Debug.Assert(j < checkOnce.Length);
Debug.Assert(checkOnce[j] == false);
checkOnce[j] = true;
}
#endif
}
#if DEBUG
for (int j = 0; j < checkOnce.Length; j++)
{
Debug.Assert(checkOnce[j] == true);
}
#endif
return(Blocks);
}
public ConvergenceObserver(MultigridOperator muop, BlockMsrMatrix MassMatrix, double[] __ExactSolution)
{
if (__ExactSolution != null)
{
if (__ExactSolution.Length != muop.BaseGridProblemMapping.LocalLength)
{
throw new ArgumentException();
}
}
this.SolverOperator = muop;
List <AggregationGridBasis[]> aggBasisSeq = new List <AggregationGridBasis[]>();
for (var mo = muop; mo != null; mo = mo.CoarserLevel)
{
aggBasisSeq.Add(mo.Mapping.AggBasis);
}
this.ExactSolution = __ExactSolution;
int[] Degrees = muop.BaseGridProblemMapping.BasisS.Select(b => b.Degree).ToArray();
BlockMsrMatrix DummyOpMatrix = new BlockMsrMatrix(muop.BaseGridProblemMapping, muop.BaseGridProblemMapping);
DummyOpMatrix.AccEyeSp(123);
MultigridOperator.ChangeOfBasisConfig[][] config = new MultigridOperator.ChangeOfBasisConfig[1][];
config[0] = new MultigridOperator.ChangeOfBasisConfig[muop.BaseGridProblemMapping.BasisS.Count];
for (int iVar = 0; iVar < config[0].Length; iVar++)
{
config[0][iVar] = new MultigridOperator.ChangeOfBasisConfig()
{
Degree = Degrees[iVar],
mode = MultigridOperator.Mode.IdMass_DropIndefinite,
VarIndex = new int[] { iVar }
};
}
//this.DecompositionOperator = muop; this.DecompositionOperator_IsOrthonormal = false;
this.DecompositionOperator = new MultigridOperator(aggBasisSeq, muop.BaseGridProblemMapping, DummyOpMatrix, MassMatrix, config); this.DecompositionOperator_IsOrthonormal = true;
ResNormTrend = new Dictionary <Tuple <int, int, int>, List <double> >();
ErrNormTrend = new Dictionary <Tuple <int, int, int>, List <double> >();
for (var mgop = this.DecompositionOperator; mgop != null; mgop = mgop.CoarserLevel)
{
int[] _Degrees = mgop.Mapping.DgDegree;
for (int iVar = 0; iVar < _Degrees.Length; iVar++)
{
for (int p = 0; p <= _Degrees[iVar]; p++)
{
ResNormTrend.Add(new Tuple <int, int, int>(mgop.LevelIndex, iVar, p), new List <double>());
ErrNormTrend.Add(new Tuple <int, int, int>(mgop.LevelIndex, iVar, p), new List <double>());
}
}
}
}
public void Init(MultigridOperator op)
{
this.m_OpThisLevel = op;
if (op.CoarserLevel == null)
{
throw new NotSupportedException("Multigrid algorithm cannot be used as a solver on the finest level.");
}
this.CoarserLevelSolver.Init(op.CoarserLevel);
}
public void Init(MultigridOperator op)
{
var Mtx = op.OperatorMatrix;
var MgMap = op.Mapping;
m_MultigridOp = op;
if (!Mtx.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!Mtx.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
m_Mtx = Mtx;
}
public void Init(MultigridOperator op)
{
BlockMsrMatrix M = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_MultigridOp = op;
if (!M.RowPartitioning.Equals(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!M.ColPartition.Equals(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
Mtx = M;
int L = M.RowPartitioning.LocalLength;
/*
* diag = new double[L];
* int i0 = Mtx.RowPartitioning.i0;
*
* for(int i = 0; i < L; i++) {
* diag[i] = Mtx[i0 + i, i0 + i];
* }
*/
if (op.Mapping.MaximalLength != op.Mapping.MinimalLength)
{
// 'BlockDiagonalMatrix' should be completely replaced by 'BlockMsrMatrix'
throw new NotImplementedException("todo - Block Jacobi for variable block Sizes");
}
int Nblk = op.Mapping.MaximalLength;
Diag = new BlockDiagonalMatrix(M.ToMsrMatrix(), Nblk, Nblk);
invDiag = Diag.Invert();
invDiag.CheckForNanOrInfM();
}
static public ISolverSmootherTemplate InitMultigridChain(MultigridOperator MgOp,
Func <int, ISolverSmootherTemplate> PreSmootherFactory,
Func <int, ISolverSmootherTemplate> PostSmootherFactory,
Action <int, ClassicMultigrid> ParamsSeter,
Func <ISolverSmootherTemplate> CoarsestSolverFactory) //
{
if (MgOp.CoarserLevel == null)
{
return(CoarsestSolverFactory());
}
else
{
var MgTop = new ClassicMultigrid();
ParamsSeter(MgOp.LevelIndex, MgTop);
MgTop.PreSmoother = PreSmootherFactory(MgOp.LevelIndex);
MgTop.PostSmoother = PostSmootherFactory(MgOp.LevelIndex);
MgTop.CoarserLevelSolver = InitMultigridChain(MgOp.CoarserLevel, PreSmootherFactory, PostSmootherFactory, ParamsSeter, CoarsestSolverFactory);
return(MgTop);
}
}
static public ISolverSmootherTemplate InitMultigridChain(MultigridOperator MgOp,
Action <int, SoftPCG> ParamsSeter,
Func <ISolverSmootherTemplate> CoarsestSolverFactory)
{
//
if (MgOp.CoarserLevel == null)
{
return(CoarsestSolverFactory());
}
else
{
var MgTop = new SoftPCG();
ParamsSeter(MgOp.LevelIndex, MgTop);
var R = new GenericRestriction();
MgTop.Precond = R;
R.CoarserLevelSolver = InitMultigridChain(MgOp.CoarserLevel, ParamsSeter, CoarsestSolverFactory);
return(MgTop);
}
}
public void Init(MultigridOperator op)
{
var Mtx = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_mgop = op;
if (!Mtx.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!Mtx.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
this.Matrix = Mtx;
if (Precond != null)
{
Precond.Init(op);
}
}
public void Init(MultigridOperator op)
{
var M = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_mgop = op;
if (!M.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!M.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
foreach (var pc in PrecondS)
{
pc.Init(m_mgop);
}
}
public void Init(MultigridOperator op)
{
var M = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_MgOp = op;
if (!M.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!M.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
this.m_Matrix = M;
m_OptMatrix = new ilPSP.LinSolvers.monkey.CPU.RefMatrix(M.ToMsrMatrix());
if (Precond != null)
{
Precond.Init(op);
}
}
/// <summary>
/// Decomposition of some solution vector <paramref name="vec"/> into the different multigrid levels.
/// </summary>
public void PlotDecomposition <V>(V vec, string plotName)
where V : IList <double> //
{
int L0 = this.DecompositionOperator.Mapping.LocalLength;
double[] Vec0 = new double[L0];
this.DecompositionOperator.TransformSolInto(vec, Vec0);
var Decomp = this.OrthonormalMultigridDecomposition(Vec0, false);
List <DGField> DecompFields = new List <DGField>();
MultigridOperator op = this.DecompositionOperator;
for (int iLevel = 0; iLevel < Decomp.Count; iLevel++)
{
double[] vec_i = Decomp[iLevel];
var DecompVec = this.InitProblemDGFields("Level" + iLevel);
MultigridOperator opi = op;
for (int k = iLevel; k > 0; k--)
{
int Lk1 = Decomp[k - 1].Length;
double[] vec_i1 = new double[Lk1];
opi.Prolongate(1.0, vec_i1, 0.0, vec_i);
vec_i = vec_i1;
opi = opi.FinerLevel;
}
this.DecompositionOperator.TransformSolFrom(DecompVec, vec_i);
DecompFields.AddRange(DecompVec.Mapping.Fields);
op = op.CoarserLevel;
}
Tecplot.Tecplot.PlotFields(DecompFields, plotName, 0.0, 3);
}
private MultigridOperator(MultigridOperator __FinerLevel, IEnumerable <AggregationGridBasis[]> basisES, UnsetteledCoordinateMapping _pm, IEnumerable <ChangeOfBasisConfig[]> cobc)
{
if (basisES.Count() <= 0)
{
throw new ArgumentException("At least one multigrid level is required.");
}
this.BaseGridProblemMapping = _pm;
if (cobc.Count() < 1)
{
throw new ArgumentException();
}
this.m_Config = cobc.First();
this.FinerLevel = __FinerLevel;
this.Mapping = new MultigridMapping(_pm, basisES.First(), this.Degrees);
if (this.Mapping.LocalLength > this.BaseGridProblemMapping.LocalLength)
{
throw new ApplicationException("Something wrong.");
}
if (basisES.Count() > 1)
{
this.CoarserLevel = new MultigridOperator(this, basisES.Skip(1), _pm, cobc.Count() > 1 ? cobc.Skip(1) : cobc);
}
if (this.LevelIndex == 0 && this.Mapping.AggBasis.Any(agb => agb.ReqModeIndexTrafo))
{
int J = this.Mapping.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
Debug.Assert(J == this.BaseGridProblemMapping.GridDat.iLogicalCells.NoOfLocalUpdatedCells);
IndexIntoProblemMapping_Local = new int[this.Mapping.LocalLength];
#if DEBUG
IndexIntoProblemMapping_Local.SetAll(-23456);
#endif
AggregationGridBasis[] AgBss = this.Mapping.AggBasis;
int[] Degrees = this.Mapping.DgDegree;
int NoFlds = Degrees.Length;
for (int j = 0; j < J; j++) // loop over cells
{
for (int iFld = 0; iFld < NoFlds; iFld++)
{
int N = AgBss[iFld].GetLength(j, Degrees[iFld]); // By using the length of the aggregate grid mapping,
// we exclude XDG agglomerated cells.
if (N > 0)
{
int k0 = this.Mapping.ProblemMapping.LocalUniqueCoordinateIndex(iFld, j, 0);
int i0 = this.Mapping.LocalUniqueIndex(iFld, j, 0);
for (int n = 0; n < N; n++)
{
//X[i0 + n] = INOUT_X[k0 + n];
//B[i0 + n] = IN_RHS[k0 + n];
int n_trf = AgBss[iFld].N_Murks(j, n, N);
Debug.Assert(n_trf >= 0);
#if DEBUG
Debug.Assert(IndexIntoProblemMapping_Local[i0 + n] < 0);
#endif
IndexIntoProblemMapping_Local[i0 + n] = k0 + n_trf;
//Debug.Assert(i0 + n == 0 || IndexIntoProblemMapping_Local[i0 + n] > IndexIntoProblemMapping_Local[i0 + n - 1]);
Debug.Assert(IndexIntoProblemMapping_Local[i0 + n] >= 0);
Debug.Assert(IndexIntoProblemMapping_Local[i0 + n] < this.Mapping.ProblemMapping.LocalLength);
}
}
else
{
// should be a cell without any species.
Debug.Assert(this.Mapping.AggBasis.Where(b => !(b is XdgAggregationBasis)).Count() == 0, "all must be XDG");
Debug.Assert(this.Mapping.AggBasis.Where(b => ((XdgAggregationBasis)b).GetNoOfSpecies(j) != 0).Count() == 0, "no species in any cell allowed");
//Debug.Assert();
}
}
}
#if DEBUG
Debug.Assert(IndexIntoProblemMapping_Local.Any(i => i < 0) == false);
#endif
if (this.Mapping.ProblemMapping.MpiSize == 1)
{
this.IndexIntoProblemMapping_Global = this.IndexIntoProblemMapping_Local;
}
else
{
this.IndexIntoProblemMapping_Global = this.IndexIntoProblemMapping_Local.CloneAs();
int i0Proc = this.Mapping.ProblemMapping.i0;
int L = this.IndexIntoProblemMapping_Global.Length;
for (int i = 0; i < L; i++)
{
this.IndexIntoProblemMapping_Global[i] += i0Proc;
}
}
}
#if DEBUG
if (IndexIntoProblemMapping_Local != null)
{
int i0 = this.Mapping.ProblemMapping.i0;
int L = this.Mapping.ProblemMapping.LocalLength;
int[] UseCount = new int[L];
foreach (int i in IndexIntoProblemMapping_Global)
{
UseCount[i - i0]++;
}
for (int l = 0; l < L; l++)
{
Debug.Assert(UseCount[l] <= 1);
}
}
#endif
}
/// <summary>
/// defines the problem matrix
/// </summary>
public void Init(MultigridOperator op)
{
this.m_MgOperator = op;
var Mtx = op.OperatorMatrix;
var MgMap = op.Mapping;
if (!Mtx.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!Mtx.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
double Dim = MgMap.ProblemMapping.GridDat.SpatialDimension;
// set operator
// ============
if (op.CoarserLevel == null)
{
throw new NotSupportedException("Multigrid algorithm cannot be used as a solver on the finest level.");
}
this.OpMatrix = Mtx;
/*
* // construct restriction and prolongation
* // ======================================
*
* var thisLevel = MgMap.AggBasis;
* var lowerLevel = MgMap.CoarserLevel.AggBasis;
*
* int[] this_p = MgMap.DgDegree;
* int[] low_p = MgMap.CoarserLevel.DgDegree;
*
* var __PrlgOperator = thisLevel.FromOtherLevelMatrix(MgMap.ProblemMapping, this_p, lowerLevel, low_p);
* Debug.Assert(__PrlgOperator.RowPartitioning.LocalLength == MgMap.LocalLength);
* Debug.Assert(__PrlgOperator.ColPartition.LocalLength == MgMap.CoarserLevel.LocalLength);
#if DEBUG
* var __RestOperator = lowerLevel.FromOtherLevelMatrix(MgMap.ProblemMapping, low_p, thisLevel, this_p);
* {
* var __RestOperator2 = __PrlgOperator.Transpose();
* __RestOperator2.Acc(-1.0, __RestOperator);
* double dingsNorm = __RestOperator2.InfNorm();
* Debug.Assert(dingsNorm <= 1.0e-10);
* }
#else
* var __RestOperator = __PrlgOperator.Transpose();
#endif
* int iLevel = MgMap.LevelIndex;
*
* if(this.UseSmoothedInterpolation) {
* MsrMatrix DinvA;
* double specRadius = Utils.rhoDinvA(OpMatrix, out DinvA);
* Console.WriteLine("Level {0} spectral radius = {1}", iLevel, specRadius);
* double omega = ((Dim + 1) / Dim) * (1.0 / specRadius);
*
* DinvA.Scale(-omega);
* //DinvA.Clear();
* DinvA.AccEyeSp(1.0);
*
* PrologateOperator = MsrMatrix.Multiply(DinvA, __PrlgOperator);
* RestrictionOperator = this.PrologateOperator.Transpose();
* } else {
* PrologateOperator = __PrlgOperator;
* RestrictionOperator = __RestOperator;
*
* var RX = this.RestrictionOperator.CloneAs();
* RX.Acc(-1.0, m_MgOperator.RestrictionOperator);
* double tst = RX.InfNorm();
* Debug.Assert(tst <= 1.0e-10);
*
* var PX = this.PrologateOperator.CloneAs();
* PX.Acc(-1.0, m_MgOperator.PrologateOperator);
* double tst2 = PX.InfNorm();
* Debug.Assert(tst2 <= 1.0e-10);
* }
* //*/
// initiate coarser level
// ======================
//var M1 = MsrMatrix.Multiply(this.OpMatrix, this.PrologateOperator);
//var CoarseMtx = MsrMatrix.Multiply(this.RestrictionOperator, M1);
if (this.CoarserLevelSolver == null)
{
throw new NotSupportedException("Missing coarse level solver.");
}
this.CoarserLevelSolver.Init(op.CoarserLevel);
// init pre&post smoother
// ======================
if (this.PreSmoother != null)
{
this.PreSmoother.Init(op);
}
if (this.PostSmoother != null)
{
this.PostSmoother.Init(op);
}
}
/// <summary> /// Number of blocs returned by <see cref="GetBlocking(MultigridOperator)"/> /// </summary> internal abstract int GetNoOfBlocks(MultigridOperator op);
/// <summary> /// Returns lists which form the blocks of the additive-Schwarz domain decomposition. /// </summary> /// <param name="op"></param> /// <returns> /// - outer enumeration: correcponts to domain-decomposition blocks /// - inner index: indices within the sub-blocks /// - content: local cell indices which form the respective additive-Schwarz block (<see cref="MultigridOperator."/> /// </returns> abstract internal IEnumerable <List <int> > GetBlocking(MultigridOperator op);
public void Init(MultigridOperator op)
{
using (new FuncTrace()) {
if (m_MgOp != null)
{
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// someone is trying to re-use this solver: see if the settings permit that
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if (op.LevelIndex != m_MgOp.LevelIndex)
{
throw new ArgumentException("Re-use on different level not possible.");
}
if (!this.MtxFull._RowPartitioning.EqualsPartition(op.OperatorMatrix._RowPartitioning))
{
throw new ArgumentException("Matrix has changed, unable to re-use");
}
if (!this.MtxFull._ColPartitioning.EqualsPartition(op.OperatorMatrix._ColPartitioning))
{
throw new ArgumentException("Matrix has changed, unable to re-use");
}
#if DEBUG
if (!object.ReferenceEquals(this.MtxFull, op.OperatorMatrix))
{
BlockMsrMatrix Check = this.MtxFull.CloneAs();
Check.Acc(-1.0, op.OperatorMatrix);
if (Check.InfNorm() != 0.0)
{
throw new ArgumentException("Matrix has changed, unable to re-use");
}
}
#endif
if (this.m_BlockingStrategy.GetNoOfBlocks(op) != this.blockSolvers.Count())
{
throw new ArgumentException("Blocking, unable to re-use");
}
return;
}
var Mop = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_MgOp = op;
int myMpiRank = MgMap.MpiRank;
int myMpisize = MgMap.MpiSize;
if (!Mop.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!Mop.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
var ag = MgMap.AggGrid;
int JComp = ag.iLogicalCells.NoOfLocalUpdatedCells;
int JGhost = ag.iLogicalCells.NoOfExternalCells;
#if DEBUG
ilPSP.Connectors.Matlab.BatchmodeConnector matlab;
if (m_MatlabParalellizationCheck)
{
matlab = new ilPSP.Connectors.Matlab.BatchmodeConnector();
}
else
{
matlab = null;
}
#endif
//Mop.Clear();
//for(int i = Mop.RowPartitioning.i0; i < Mop.RowPartitioning.iE; i++) {
// Mop[i, i] = i + 1;
//}
// get cell blocks
// ===============
var _Blocks = this.m_BlockingStrategy.GetBlocking(op);
int NoOfSchwzBlocks = _Blocks.Count();
// test cell blocks
// ================
#if DEBUG
{
// ensure that each cell is used exactly once, among all blocks
bool[] test = new bool[ag.iLogicalCells.NoOfLocalUpdatedCells];
foreach (var bi in _Blocks)
{
foreach (int j in bi)
{
Debug.Assert(test[j] == false);
test[j] = true;
}
;
}
for (int i = 0; i < test.Length; i++)
{
Debug.Assert(test[i] == true);
}
}
#endif
// extend blocks according to desired overlap
// ==========================================
{
BitArray marker = new BitArray(JComp + JGhost);
if (Overlap < 0)
{
throw new ArgumentException();
}
if (Overlap > 0)
{
if (Overlap > 1 && Mop.RowPartitioning.MpiSize > 1)
{
throw new NotSupportedException("In MPI parallel runs, the maximum supported overlap for the Schwarz preconditioner is 1.");
}
foreach (List <int> bi in _Blocks) // loop over blocks...
{
marker.SetAll(false); // marks all cells which are members of the block
foreach (int jcomp in bi)
{
marker[jcomp] = true;
}
// determine overlap regions
for (int k = 0; k < Overlap; k++)
{
int Jblock = bi.Count;
for (int j = 0; j < Jblock; j++)
{
int jCell = bi[j];
int[] Neighs = ag.iLogicalCells.CellNeighbours[jCell];
foreach (int jNeigh in Neighs)
{
if (marker[jNeigh] == false)
{
// neighbor cell is not already a member of the block
// => add it.
bi.Add(jNeigh);
marker[jNeigh] = true;
}
}
}
}
bi.Sort();
}
}
BlockCells = _Blocks.Select(list => list.ToArray()).ToArray();
}
// convert cell blocks to DOF blocks
// =================================
List <int>[] BlkIdx_gI_lR; // for each Schwarz block, (global) indices in the local range
List <int>[] BlkIdx_gI_eR; // for each Schwarz block, (global) indices of external rows and columns
List <int>[] TempRowIdx_gI; // for each Schwarz block, (global) indices into the temporary matrix
List <int>[] BlkIdx_lI_eR; // for each Schwarz block, (local) indices of external rows and columns
List <int>[] LocalBlocks_i0, LocalBlocks_N; // blocking of the Schwarz-Blocks.
// for matrix 'ExternalRowsTemp': which rows of 'Mop' are required locally
List <int> ExternalRowsIndices, ExternalRows_BlockI0, ExternalRows_BlockN;
{
int Jup = MgMap.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
int Jgh = MgMap.AggGrid.iLogicalCells.NoOfExternalCells;
int LocalizedBlockCounter = 0;
BlkIdx_gI_lR = NoOfSchwzBlocks.ForLoop(iPart => new List <int>(BlockCells[iPart].Length * MgMap.MaximalLength));
BlkIdx_gI_eR = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
LocalBlocks_i0 = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
LocalBlocks_N = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
TempRowIdx_gI = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
BlkIdx_lI_eR = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
ExternalRowsIndices = new List <int>();
ExternalRows_BlockI0 = new List <int>();
ExternalRows_BlockN = new List <int>();
for (int iPart = 0; iPart < NoOfSchwzBlocks; iPart++) // loop over parts...
{
int[] bc = BlockCells[iPart];
var biI = BlkIdx_gI_lR[iPart];
var biE = BlkIdx_gI_eR[iPart];
var l1 = TempRowIdx_gI[iPart];
var l2 = BlkIdx_lI_eR[iPart];
var LBBi0 = LocalBlocks_i0[iPart];
var LBBN = LocalBlocks_N[iPart];
int Jblock = bc.Length;
int anotherCounter = 0;
for (int jblk = 0; jblk < Jblock; jblk++) // loop over cells in blocks...
{
int j = bc[jblk];
int N = MgMap.GetLength(j);
if (j < Jup)
{
// locally updated cell
int i0 = MgMap.GlobalUniqueIndex(0, j, 0);
for (int n = 0; n < N; n++)
{
biI.Add(i0 + n);
}
}
else
{
// external cell
int i0E = MgMap.GlobalUniqueIndex(0, j, 0); //
int i0L = MgMap.LocalUniqueIndex(0, j, 0); //
ExternalRows_BlockI0.Add(LocalizedBlockCounter);
ExternalRows_BlockN.Add(N);
//LEBi0.Add(LocalizedBlockCounter);
//LEBn.Add(N);
for (int n = 0; n < N; n++)
{
biE.Add(i0E + n);
ExternalRowsIndices.Add(i0E + n);
l1.Add(LocalizedBlockCounter + n);
l2.Add(i0L + n);
Debug.Assert(Mop._RowPartitioning.FindProcess(i0E + n) != myMpiRank);
}
LocalizedBlockCounter += N;
}
LBBi0.Add(anotherCounter);
LBBN.Add(N);
anotherCounter += N;
}
}
//this.BlockIndices = _LocallyStoredBlockIndices.Select(bi => bi.ToArray()).ToArray();
}
// get rows for blocks that use external cells
// ===========================================
#if DEBUG
{
if (Overlap == 0)
{
Debug.Assert(ExternalRowsIndices.Count == 0);
Debug.Assert(ExternalRows_BlockI0.Count == 0);
Debug.Assert(ExternalRows_BlockN.Count == 0);
}
foreach (var bi in BlkIdx_gI_lR)
{
foreach (int idx in bi)
{
Debug.Assert(idx >= m_MgOp.Mapping.i0);
Debug.Assert(idx < m_MgOp.Mapping.iE);
}
}
foreach (var ei in BlkIdx_gI_eR)
{
foreach (int idx in ei)
{
Debug.Assert(idx < m_MgOp.Mapping.i0 || idx >= m_MgOp.Mapping.iE);
}
}
int LL = m_MgOp.Mapping.LocalLength;
int jMax = m_MgOp.Mapping.AggGrid.iLogicalCells.NoOfCells - 1;
int LE = m_MgOp.Mapping.LocalUniqueIndex(0, jMax, 0) + m_MgOp.Mapping.GetLength(jMax);
foreach (var ci in BlkIdx_lI_eR)
{
foreach (int idx in ci)
{
Debug.Assert(idx >= LL);
Debug.Assert(idx < LE);
}
}
if (m_MatlabParalellizationCheck)
{
int globalBlockCounter = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
if (rankCounter == myMpiRank)
{
Debug.Assert(rank_NoBlks == NoOfSchwzBlocks);
}
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
double[] vec;
if (rankCounter == myMpiRank)
{
vec = ArrayTools.Cat(BlkIdx_gI_lR[iBlock], BlkIdx_gI_eR[iBlock]).Select(ii => ((double)(ii + 1))).ToArray();
}
else
{
vec = new double[0];
}
matlab.PutVector(vec, string.Format("BlockIdx{0}", globalBlockCounter));
globalBlockCounter++;
csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD);
}
}
}
}
#endif
BlockMsrMatrix ExternalRowsTemp;
if (myMpisize > 1 && Overlap > 0)
{
//int NoOfLocalRows = _ExternalBlockIndices.Sum(L => L.Count);
BlockPartitioning PermRow = new BlockPartitioning(ExternalRowsIndices.Count, ExternalRows_BlockI0, ExternalRows_BlockN, Mop.MPI_Comm, i0isLocal: true);
// Remark: we use a permutation matrix for MPI-exchange of rows
BlockMsrMatrix Perm = new BlockMsrMatrix(PermRow, Mop._RowPartitioning);
for (int iRow = 0; iRow < ExternalRowsIndices.Count; iRow++)
{
Debug.Assert(Mop._RowPartitioning.IsInLocalRange(ExternalRowsIndices[iRow]) == false);
Perm[iRow + PermRow.i0, ExternalRowsIndices[iRow]] = 1;
}
ExternalRowsTemp = BlockMsrMatrix.Multiply(Perm, Mop);
#if DEBUG
if (m_MatlabParalellizationCheck)
{
matlab.PutSparseMatrix(Perm, "Perm");
matlab.PutSparseMatrix(ExternalRowsTemp, "ExternalRowsTemp");
}
#endif
}
else
{
ExternalRowsTemp = null;
}
ExternalRowsIndices = null;
ExternalRows_BlockI0 = null;
ExternalRows_BlockN = null;
// create solvers
// ==============
{
blockSolvers = new ISparseSolver[NoOfSchwzBlocks];
#if DEBUG
List <BlockMsrMatrix> Blocks = new List <BlockMsrMatrix>();
#endif
for (int iPart = 0; iPart < NoOfSchwzBlocks; iPart++)
{
var bi = BlkIdx_gI_lR[iPart];
int Bsz;
if (MgMap.MinimalLength == MgMap.MaximalLength)
{
Bsz = MgMap.MaximalLength;
}
else
{
Bsz = 1;
}
var l1 = TempRowIdx_gI[iPart];
//if (M.RowPartitioning.MpiSize > 1) {
// int i0Proc = M.RowPartitioning.i0;
// bi = bi.CloneAs();
// for (int i = 0; i < bi.Length; i++) {
// bi[i] += i0Proc;
// }
//}
BlockPartitioning localBlocking = new BlockPartitioning(bi.Count + l1.Count, LocalBlocks_i0[iPart], LocalBlocks_N[iPart], csMPI.Raw._COMM.SELF);
if (l1.Count > 0)
{
// convert the indices into 'ExternalRowsTemp' to global indices
int l1L = l1.Count;
int offset = ExternalRowsTemp._RowPartitioning.i0;
for (int i = 0; i < l1L; i++)
{
l1[i] += offset;
}
}
BlockMsrMatrix Block = new BlockMsrMatrix(localBlocking, localBlocking);// bi.Length, bi.Length, Bsz, Bsz);
Mop.WriteSubMatrixTo(Block, bi, default(int[]), bi, default(int[]));
if (l1.Count > 0)
{
int offset = bi.Count;
int[] targRows = l1.Count.ForLoop(i => i + offset);
var biE = BlkIdx_gI_eR[iPart];
int[] extTargCols = biE.Count.ForLoop(i => i + offset);
Mop.AccSubMatrixTo(1.0, Block, bi, default(int[]), new int[0], default(int[]), biE, extTargCols);
ExternalRowsTemp.AccSubMatrixTo(1.0, Block, l1, targRows, bi, default(int[]), biE, extTargCols);
}
#if DEBUG
if (m_MatlabParalellizationCheck)
{
Blocks.Add(Block);
}
#endif
blockSolvers[iPart] = new PARDISOSolver()
{
CacheFactorization = true,
UseDoublePrecision = false
};
//blockSolvers[iPart] = new FullDirectSolver();
//blockSolvers[iPart] = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
blockSolvers[iPart].DefineMatrix(Block);
}
#if DEBUG
if (m_MatlabParalellizationCheck)
{
int globalBlockCounter = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
BlockMsrMatrix Block;
if (rankCounter == myMpiRank)
{
Block = Blocks[iBlock];
}
else
{
Block = null;
}
matlab.PutSparseMatrix(Block, string.Format("Block{0}", globalBlockCounter));
globalBlockCounter++;
csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD);
}
}
}
#endif
}
// Record required indices
// =======================
{
this.BlockIndices_Local = new int[NoOfSchwzBlocks][];
this.BlockIndices_External = new int[NoOfSchwzBlocks][];
int LocalI0 = MgMap.i0;
int LocalLength = MgMap.LocalLength;
for (int iBlock = 0; iBlock < NoOfSchwzBlocks; iBlock++)
{
var _bi = BlkIdx_gI_lR[iBlock];
int L = _bi.Count;
int[] bil = new int[L];
this.BlockIndices_Local[iBlock] = bil;
for (int l = 0; l < L; l++)
{
bil[l] = _bi[l] - LocalI0;
Debug.Assert(bil[l] >= 0);
Debug.Assert(bil[l] < MgMap.LocalLength);
}
var _biE = BlkIdx_lI_eR[iBlock];
if (_biE.Count > 0)
{
this.BlockIndices_External[iBlock] = _biE.ToArray();
}
}
}
this.MtxFull = Mop;
if (CoarseSolver != null)
{
CoarseSolver.Init(op.CoarserLevel);
}
// Debug & Test-Code
// =================
#if DEBUG
if (m_MatlabParalellizationCheck)
{
Console.WriteLine("Matlab dir: " + matlab.WorkingDirectory);
matlab.PutSparseMatrix(Mop, "Full");
int GlobalNoOfBlocks = NoOfSchwzBlocks.MPISum();
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
matlab.Cmd("BlockErr({0} + 1, 1) = norm( Block{0} - Full( BlockIdx{0}, BlockIdx{0} ), inf );", iGlbBlock);
}
Random rnd = new Random(myMpiRank);
double[] testRHS = new double[MgMap.LocalLength];
for (int i = 0; i < testRHS.Length; i++)
{
testRHS[i] = rnd.NextDouble();
}
matlab.PutVector(testRHS, "testRHS");
MPIexchange <double[]> ResExchange = new MPIexchange <double[]>(MgMap, testRHS);
ResExchange.TransceiveStartImReturn();
ResExchange.TransceiveFinish(0.0);
int offset = MgMap.LocalLength;
int g = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
double[] SubVec;
if (rankCounter == myMpiRank)
{
int LL = this.BlockIndices_Local[iBlock].Length;
int LE;
if (this.BlockIndices_External[iBlock] != null)
{
LE = this.BlockIndices_External[iBlock].Length;
}
else
{
LE = 0;
}
int L = LL + LE;
SubVec = new double[L];
for (int i = 0; i < LL; i++)
{
SubVec[i] = testRHS[this.BlockIndices_Local[iBlock][i]];
}
if (LE > 0)
{
for (int i = 0; i < LE; i++)
{
SubVec[i + LL] = ResExchange.Vector_Ext[this.BlockIndices_External[iBlock][i] - offset];
}
}
}
else
{
SubVec = new double[0];
}
matlab.PutVector(SubVec, "SubVec" + g);
g++;
}
}
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
matlab.Cmd("RhsErr({0} + 1, 1) = norm( SubVec{0} - testRHS( BlockIdx{0} ), inf );", iGlbBlock);
}
double[] testX = new double[testRHS.Length];
MPIexchangeInverse <double[]> XExchange = new MPIexchangeInverse <double[]>(MgMap, testX);
g = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
if (rankCounter == myMpiRank)
{
int LL = this.BlockIndices_Local[iBlock].Length;
int LE;
if (this.BlockIndices_External[iBlock] != null)
{
LE = this.BlockIndices_External[iBlock].Length;
}
else
{
LE = 0;
}
int L = LL + LE;
for (int i = 0; i < LL; i++)
{
testX[this.BlockIndices_Local[iBlock][i]] += (g + 1);
}
if (LE > 0)
{
for (int i = 0; i < LE; i++)
{
XExchange.Vector_Ext[this.BlockIndices_External[iBlock][i] - offset] += (g + 1);
}
}
}
else
{
//nop
}
g++;
}
}
XExchange.TransceiveStartImReturn();
XExchange.TransceiveFinish(1.0);
matlab.Cmd("testXref = zeros({0},1);", MgMap.TotalLength);
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
matlab.Cmd("testXref(BlockIdx{0},1) = testXref(BlockIdx{0},1) + ({0} + 1);", iGlbBlock);
}
matlab.PutVector(testX, "testX");
matlab.Cmd("testXErr = norm(testX - testXref, inf);");
MultidimensionalArray BlockErr = MultidimensionalArray.Create(GlobalNoOfBlocks, 1);
MultidimensionalArray RhsErr = MultidimensionalArray.Create(GlobalNoOfBlocks, 1);
MultidimensionalArray testXErr = MultidimensionalArray.Create(1, 1);
matlab.GetMatrix(BlockErr, "BlockErr");
matlab.GetMatrix(RhsErr, "RhsErr");
matlab.GetMatrix(testXErr, "testXErr");
matlab.Execute();
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
Console.WriteLine("Block #{0} Error (external? ) " + BlockErr[iGlbBlock, 0], iGlbBlock);
Console.WriteLine("RHS #{0} Error " + RhsErr[iGlbBlock, 0], iGlbBlock);
Debug.Assert(BlockErr[iGlbBlock, 0] == 0);
Debug.Assert(RhsErr[iGlbBlock, 0] == 0);
}
Console.WriteLine("X Error " + testXErr[0, 0]);
Debug.Assert(testXErr[0, 0] == 0.0);
matlab.Dispose();
}
#endif
}
}
/// <summary>
/// %
/// </summary>
internal override int GetNoOfBlocks(MultigridOperator op)
{
return(NoOfPartsPerProcess);
}
internal override IEnumerable <List <int> > GetBlocking(MultigridOperator op)
{
var MgMap = op.Mapping;
//if(!M.RowPartitioning.Equals(MgMap.Partitioning))
// throw new ArgumentException("Row partitioning mismatch.");
//if(!M.ColPartition.Equals(MgMap.Partitioning))
// throw new ArgumentException("Column partitioning mismatch.");
var ag = MgMap.AggGrid;
int JComp = ag.iLogicalCells.NoOfLocalUpdatedCells; // number of local cells
int[] xadj = new int[JComp + 1];
List <int> adjncy = new List <int>();
for (int j = 0; j < JComp; j++)
{
int[] neigh_j = ag.iLogicalCells.CellNeighbours[j];
foreach (int jNeigh in neigh_j)
{
//adjncy.AddRange(neigh_j);
if (jNeigh < JComp)
{
adjncy.Add(jNeigh);
}
else
{
//Console.WriteLine("Skipping external cell");
}
}
xadj[j + 1] = xadj[j] + neigh_j.Length;
}
int MPIrank, MPIsize;
MPI.Wrappers.csMPI.Raw.Comm_Rank(MPI.Wrappers.csMPI.Raw._COMM.WORLD, out MPIrank);
MPI.Wrappers.csMPI.Raw.Comm_Size(MPI.Wrappers.csMPI.Raw._COMM.WORLD, out MPIsize);
//if (MPIrank == 1)
// NoOfParts = 1;
//Debugger.Launch();
int[] part = new int[JComp];
{
if (NoOfPartsPerProcess > 1)
{
int ncon = 1;
int edgecut = 0;
int[] options = null; //new int[] { 0, 0, 0, 0, 0 };
METIS.PartGraphKway(
ref JComp, ref ncon,
xadj,
adjncy.ToArray(),
null,
null,
null,
ref NoOfPartsPerProcess,
null,
null,
options,
ref edgecut,
part);
}
else
{
part.SetAll(0);
}
}
{
var _Blocks = NoOfPartsPerProcess.ForLoop(i => new List <int>((int)Math.Ceiling(1.1 * JComp / NoOfPartsPerProcess)));
for (int j = 0; j < JComp; j++)
{
_Blocks[part[j]].Add(j);
}
return(_Blocks);
}
}
/// <summary>
/// %
/// </summary>
internal override int GetNoOfBlocks(MultigridOperator op)
{
return(GetBlocking(op).Count());
}
/// <summary>
/// Triggers <see cref="IterationCallback"/>.
/// </summary>
protected void OnIterationCallback(int iterIndex, double[] currentSol, double[] currentRes, MultigridOperator Mgop)
{
if (IterationCallback != null)
{
IterationCallback(iterIndex, currentSol, currentRes, Mgop);
}
}
public void Init(MultigridOperator op)
{
int D = op.GridData.SpatialDimension;
CodName = (new string[] { "mom0", "mom1" });
Params = ArrayTools.Cat(
VariableNames.Velocity0Vector(D));
DomName = ArrayTools.Cat(VariableNames.VelocityVector(D));
LocalOp = new SpatialOperator(DomName, Params, CodName, (A, B, C) => 4);
for (int d = 0; d < D; d++)
{
LocalOp.EquationComponents["mom" + d].Add(new LocalDiffusiveFlux()
{
m_component = d, dt = m_dt, muA = m_muA
});
}
LocalOp.Commit();
//LocalMatrix = op.MassMatrix.CloneAs().ToMsrMatrix();
//LocalMatrix.Clear();
//var U0 = new VectorField<SinglePhaseField>(op.BaseGridProblemMapping Take(D).Select(F => (SinglePhaseField)F).ToArray());
UnsetteledCoordinateMapping test = new UnsetteledCoordinateMapping(op.BaseGridProblemMapping.BasisS.GetSubVector(0, D));
var U0 = ((BoSSS.Foundation.CoordinateMapping)op.Mapping.ProblemMapping).Fields.GetSubVector(0, 2);
var empty = new SinglePhaseField[D];
LocalMatrix = LocalOp.ComputeMatrix(test, empty, test, time: m_dt);
Uidx = op.Mapping.ProblemMapping.GetSubvectorIndices(true, D.ForLoop(i => i));
Pidx = op.Mapping.ProblemMapping.GetSubvectorIndices(true, D);
int Upart = Uidx.Length;
int Ppart = Pidx.Length;
ConvDiff = new MsrMatrix(Upart, Upart, 1, 1);
pGrad = new MsrMatrix(Upart, Ppart, 1, 1);
divVel = new MsrMatrix(Ppart, Upart, 1, 1);
var VelocityMass = new MsrMatrix(Upart, Upart, 1, 1);
var leftChangeBasesVel = new MsrMatrix(Upart, Upart, 1, 1);
var rightChangeBasesVel = new MsrMatrix(Upart, Upart, 1, 1);
op.MassMatrix.AccSubMatrixTo(1.0, VelocityMass, Uidx, default(int[]), Uidx, default(int[]), default(int[]), default(int[]));
op.LeftChangeOfBasis.AccSubMatrixTo(1.0, leftChangeBasesVel, Uidx, default(int[]), Uidx, default(int[]), default(int[]), default(int[]));
op.RightChangeOfBasis.AccSubMatrixTo(1.0, rightChangeBasesVel, Uidx, default(int[]), Uidx, default(int[]), default(int[]), default(int[]));
var temp = MsrMatrix.Multiply(leftChangeBasesVel, LocalMatrix);
LocalMatrix = MsrMatrix.Multiply(temp, rightChangeBasesVel);
var M = op.OperatorMatrix;
M.AccSubMatrixTo(1.0, ConvDiff, Uidx, default(int[]), Uidx, default(int[]), default(int[]), default(int[]));
M.AccSubMatrixTo(1.0, pGrad, Uidx, default(int[]), Pidx, default(int[]), default(int[]), default(int[]));
M.AccSubMatrixTo(1.0, divVel, Pidx, default(int[]), Uidx, default(int[]), default(int[]), default(int[]));
LocalMatrix.SaveToTextFileSparse("LocalConvDiffMatrix");
ConvDiff.SaveToTextFileSparse("ConvDiff");
op.MassMatrix.SaveToTextFileSparse("MassMatrix");
VelocityMass.SaveToTextFileSparse("VelocityMass");
}
/// <summary>
/// Callback routine, see <see cref="ISolverWithCallback.IterationCallback"/> or <see cref="NonlinearSolver.IterationCallback"/>.
/// </summary>
public void IterationCallback(int iter, double[] xI, double[] rI, MultigridOperator mgOp)
{
if (xI.Length != SolverOperator.Mapping.LocalLength)
{
throw new ArgumentException();
}
if (rI.Length != SolverOperator.Mapping.LocalLength)
{
throw new ArgumentException();
}
int Lorg = SolverOperator.BaseGridProblemMapping.LocalLength;
// transform residual and solution back onto the orignal grid
// ==========================================================
double[] Res_Org = new double[Lorg];
double[] Sol_Org = new double[Lorg];
SolverOperator.TransformRhsFrom(Res_Org, rI);
SolverOperator.TransformSolFrom(Sol_Org, xI);
double[] Err_Org = Sol_Org.CloneAs();
Err_Org.AccV(-1.0, this.ExactSolution);
if (TecplotOut != null)
{
var ErrVec = InitProblemDGFields("Err");
var ResVec = InitProblemDGFields("Res");
var SolVec = InitProblemDGFields("Sol");
ErrVec.SetV(Err_Org);
ResVec.SetV(Res_Org);
SolVec.SetV(Sol_Org);
List <DGField> ErrResSol = new List <DGField>();
ErrResSol.AddRange(ErrVec.Mapping.Fields);
ErrResSol.AddRange(ResVec.Mapping.Fields);
ErrResSol.AddRange(SolVec.Mapping.Fields);
Tecplot.Tecplot.PlotFields(ErrResSol, TecplotOut + "." + iter, iter, 4);
}
// Console out
// ===========
double l2_RES = rI.L2NormPow2().MPISum().Sqrt();
double l2_ERR = Err_Org.L2NormPow2().MPISum().Sqrt();
Console.WriteLine("Iter: {0}\tRes: {1:0.##E-00}\tErr: {2:0.##E-00}", iter, l2_RES, l2_ERR);
// decompose error and residual into orthonormal vectors
// =====================================================
int L0 = DecompositionOperator.Mapping.LocalLength;
double[] Err_0 = new double[L0], Res_0 = new double[L0];
DecompositionOperator.TransformSolInto(Err_Org, Err_0);
DecompositionOperator.TransformRhsInto(Res_Org, Res_0);
IList <double[]> Err_OrthoLevels = OrthonormalMultigridDecomposition(Err_0);
IList <double[]> Res_OrthoLevels = OrthonormalMultigridDecomposition(Res_0);
// compute L2 norms on each level
// ==============================
for (var mgop = this.DecompositionOperator; mgop != null; mgop = mgop.CoarserLevel)
{
int[] _Degrees = mgop.Mapping.DgDegree;
double[] Resi = Res_OrthoLevels[mgop.LevelIndex];
double[] Errr = Err_OrthoLevels[mgop.LevelIndex];
int JAGG = mgop.Mapping.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
for (int iVar = 0; iVar < _Degrees.Length; iVar++)
{
for (int p = 0; p <= _Degrees[iVar]; p++)
{
List <double> ResNorm = this.ResNormTrend[new Tuple <int, int, int>(mgop.LevelIndex, iVar, p)];
List <double> ErrNorm = this.ErrNormTrend[new Tuple <int, int, int>(mgop.LevelIndex, iVar, p)];
double ResNormAcc = 0.0;
double ErrNormAcc = 0.0;
for (int jagg = 0; jagg < JAGG; jagg++)
{
int[] NN = mgop.Mapping.AggBasis[iVar].ModeIndexForDegree(jagg, p, _Degrees[iVar]);
foreach (int n in NN)
{
int idx = mgop.Mapping.LocalUniqueIndex(iVar, jagg, n);
ResNormAcc += Resi[idx].Pow2();
ErrNormAcc += Errr[idx].Pow2();
}
}
ResNorm.Add(ResNormAcc.Sqrt());
ErrNorm.Add(ErrNormAcc.Sqrt());
}
}
}
}
public void Init(MultigridOperator op)
{
int D = op.Mapping.GridData.SpatialDimension;
var M = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_mgop = op;
if (!M.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!M.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
Uidx = MgMap.ProblemMapping.GetSubvectorIndices(true, D.ForLoop(i => i));
Pidx = MgMap.ProblemMapping.GetSubvectorIndices(true, D);
int Upart = Uidx.Length;
int Ppart = Pidx.Length;
ConvDiff = new MsrMatrix(Upart, Upart, 1, 1);
pGrad = new MsrMatrix(Upart, Ppart, 1, 1);
divVel = new MsrMatrix(Ppart, Upart, 1, 1);
var PxP = new MsrMatrix(Ppart, Ppart, 1, 1);
M.AccSubMatrixTo(1.0, ConvDiff, Uidx, default(int[]), Uidx, default(int[]));
M.AccSubMatrixTo(1.0, pGrad, Uidx, default(int[]), Pidx, default(int[]));
M.AccSubMatrixTo(1.0, divVel, Pidx, default(int[]), Uidx, default(int[]));
M.AccSubMatrixTo(1.0, PxP, Pidx, default(int[]), Pidx, default(int[]));
Mtx = M;
int L = M.RowPartitioning.LocalLength;
int i0 = Mtx.RowPartitioning.i0;
P = new MsrMatrix(Mtx);
P.Clear();
// Debugging output
//ConvDiff.SaveToTextFileSparse("ConvDiff");
//divVel.SaveToTextFileSparse("divVel");
//pGrad.SaveToTextFileSparse("pGrad");
//PxP.SaveToTextFileSparse("PxP");
velMassMatrix = new MsrMatrix(Upart, Upart, 1, 1);
op.MassMatrix.AccSubMatrixTo(1.0, velMassMatrix, Uidx, default(int[]), Uidx, default(int[]));
switch (SchurOpt)
{
case SchurOptions.exact:
{
// Building complete Schur and Approximate Schur
MultidimensionalArray Poisson = MultidimensionalArray.Create(Pidx.Length, Pidx.Length);
MultidimensionalArray SchurConvPart = MultidimensionalArray.Create(Pidx.Length, Pidx.Length);
MultidimensionalArray Schur = MultidimensionalArray.Create(Pidx.Length, Pidx.Length);
using (BatchmodeConnector bmc = new BatchmodeConnector())
{
bmc.PutSparseMatrix(ConvDiff, "ConvDiff");
bmc.PutSparseMatrix(velMassMatrix, "MassMatrix");
bmc.PutSparseMatrix(divVel, "divVel");
bmc.PutSparseMatrix(pGrad, "pGrad");
bmc.Cmd("Qdiag = diag(diag(MassMatrix))");
bmc.Cmd("invT= inv(Qdiag)");
bmc.Cmd("Poisson = full(invT)*pGrad");
bmc.Cmd("ConvPart = ConvDiff*Poisson");
bmc.Cmd("ConvPart= full(invT)*ConvPart");
bmc.Cmd("ConvPart= divVel*ConvPart");
bmc.Cmd("Poisson = divVel*Poisson");
bmc.Cmd("ConvDiffInv = inv(full(ConvDiff))");
bmc.Cmd("Schur = divVel*ConvDiffInv");
bmc.Cmd("Schur = Schur*pGrad");
bmc.GetMatrix(Poisson, "Poisson");
bmc.GetMatrix(SchurConvPart, "ConvPart");
bmc.GetMatrix(Schur, "-Schur");
bmc.Execute(false);
}
PoissonMtx_T = Poisson.ToMsrMatrix();
PoissonMtx_H = Poisson.ToMsrMatrix();
SchurConvMtx = SchurConvPart.ToMsrMatrix();
SchurMtx = Schur.ToMsrMatrix();
SchurMtx.Acc(PxP, 1);
ConvDiff.AccSubMatrixTo(1.0, P, default(int[]), Uidx, default(int[]), Uidx);
pGrad.AccSubMatrixTo(1.0, P, default(int[]), Uidx, default(int[]), Pidx);
SchurMtx.AccSubMatrixTo(1.0, P, default(int[]), Pidx, default(int[]), Pidx);
return;
}
case SchurOptions.decoupledApprox:
{
// Do assembly for approximate Schur inverse
invVelMassMatrix = velMassMatrix.CloneAs();
invVelMassMatrix.Clear();
invVelMassMatrixSqrt = invVelMassMatrix.CloneAs();
for (int i = velMassMatrix.RowPartitioning.i0; i < velMassMatrix.RowPartitioning.iE; i++)
{
if (ApproxScaling)
{
invVelMassMatrix.SetDiagonalElement(i, 1 / (velMassMatrix[i, i]));
invVelMassMatrixSqrt.SetDiagonalElement(i, 1 / (Math.Sqrt(velMassMatrix[i, i])));
}
else
{
invVelMassMatrix.SetDiagonalElement(i, 1);
invVelMassMatrixSqrt.SetDiagonalElement(i, 1);
}
}
//invVelMassMatrix.SaveToTextFileSparse("invVelMassMatrix");
//velMassMatrix.SaveToTextFileSparse("velMassMatrix");
//ConvDiffPoissonMtx = MsrMatrix.Multiply(ConvDiff, pGrad);
//ConvDiffPoissonMtx = MsrMatrix.Multiply(divVel, ConvDiffPoissonMtx);
// Inverse of mass matrix in Matlab
//MultidimensionalArray temp = MultidimensionalArray.Create(Uidx.Length, Uidx.Length);
//using (BatchmodeConnector bmc = new BatchmodeConnector())
//{
// bmc.PutSparseMatrix(velMassMatrix, "velMassMatrix");
// bmc.Cmd("invVelMassMatrix = inv(full(velMassMatrix))");
// bmc.GetMatrix(temp, "invVelMassMatrix");
// bmc.Execute(false);
//}
//invVelMassMatrix = temp.ToMsrMatrix();
//ConvDiffPoissonMtx = MsrMatrix.Multiply(ConvDiffPoissonMtx, PoissonMtx);
//ConvDiffPoissonMtx = MsrMatrix.Multiply(PoissonMtx, ConvDiffPoissonMtx);
//ConvDiff.AccSubMatrixTo(1.0, P, default(int[]), Uidx, default(int[]), Uidx);
//pGrad.AccSubMatrixTo(1.0, P, default(int[]), Uidx, default(int[]), Pidx);
//ConvDiffPoissonMtx.AccSubMatrixTo(1.0, P, default(int[]), Pidx, default(int[]), Pidx);
//op.MassMatrix.SaveToTextFileSparse("MassMatrix");
//velMassMatrix.SaveToTextFileSparse("velMassMatrix2");
// Possion scaled by inverse of the velocity mass matrix
PoissonMtx_T = MsrMatrix.Multiply(invVelMassMatrix, pGrad);
PoissonMtx_T = MsrMatrix.Multiply(divVel, PoissonMtx_T);
////PoissonMtx_T.Acc(PxP, 1); // p.379
// Poisson scaled by sqrt of inverse of velocity mass matrix
PoissonMtx_H = MsrMatrix.Multiply(invVelMassMatrixSqrt, pGrad);
PoissonMtx_H = MsrMatrix.Multiply(divVel, PoissonMtx_H);
//PoissonMtx_H.Acc(PxP, 1); // p.379
return;
}
case SchurOptions.SIMPLE:
{
var invdiag_ConvDiff = ConvDiff.CloneAs();
invdiag_ConvDiff.Clear();
for (int i = ConvDiff.RowPartitioning.i0; i < ConvDiff.RowPartitioning.iE; i++)
{
invdiag_ConvDiff[i, i] = 1 / ConvDiff[i, i];
}
simpleSchur = MsrMatrix.Multiply(invdiag_ConvDiff, pGrad);
simpleSchur = MsrMatrix.Multiply(divVel, simpleSchur);
return;
}
default:
throw new NotImplementedException("SchurOption");
}
//var ConvDiffInvMtx = ConvDiffInv.ToMsrMatrix();
//// x= inv(P)*b !!!!! To be done with approximate Inverse
// P.SpMV(1, B, 0, X);
}
public void Init(MultigridOperator op)
{
var M = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_MgOp = op;
if (!M.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!M.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
var ag = MgMap.AggGrid;
int JComp = ag.iLogicalCells.NoOfLocalUpdatedCells;
int JGhost = ag.iLogicalCells.NoOfExternalCells;
// get cell blocks
// ===============
var _Blocks = this.m_BlockingStrategy.GetBlocking(op);
int NoParts = _Blocks.Count();
// test cell blocks
// ================
#if DEBUG
{
// ensure that each cell is used exactly once, among all blocks
bool[] test = new bool[ag.iLogicalCells.NoOfLocalUpdatedCells];
foreach (var bi in _Blocks)
{
foreach (int j in bi)
{
Debug.Assert(test[j] == false);
test[j] = true;
}
;
}
for (int i = 0; i < test.Length; i++)
{
Debug.Assert(test[i] == true);
}
}
#endif
// extend blocks according to desired overlap
// ==========================================
{
BitArray marker = new BitArray(JComp + JGhost);
if (overlap < 0)
{
throw new ArgumentException();
}
if (overlap > 0)
{
foreach (List <int> bi in _Blocks) // loop over blocks...
{
marker.SetAll(false); // marks all cells which are members of the block
foreach (int jcomp in bi)
{
marker[jcomp] = true;
}
// determine overlap regions
for (int k = 0; k < overlap; k++)
{
int Jblock = bi.Count;
for (int j = 0; j < Jblock; j++)
{
int jCell = bi[j];
int[] Neighs = ag.iLogicalCells.CellNeighbours[jCell];
foreach (int jNeigh in Neighs)
{
if (marker[jNeigh] == false)
{
// neighbor cell is not already a member of the block
// => add it.
bi.Add(jNeigh);
marker[jNeigh] = true;
}
}
}
}
bi.Sort();
}
}
BlockCells = _Blocks.Select(list => list.ToArray()).ToArray();
}
// convert cell blocks to DOF blocks
// =================================
{
int Jup = MgMap.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
int Jgh = MgMap.AggGrid.iLogicalCells.NoOfExternalCells;
var _BlockIndices = NoParts.ForLoop(iPart => new List <int>(BlockCells[iPart].Length * MgMap.MaximalLength));
for (int iPart = 0; iPart < NoParts; iPart++) // loop over parts...
{
int[] bc = BlockCells[iPart];
var bi = _BlockIndices[iPart];
int Jblock = bc.Length;
for (int jblk = 0; jblk < Jblock; jblk++)
{
int j = bc[jblk];
if (j < Jup)
{
int N = MgMap.GetLength(j);
int i0 = MgMap.LocalUniqueIndex(0, j, 0);
for (int n = 0; n < N; n++)
{
bi.Add(i0 + n);
}
}
else
{
throw new NotImplementedException("todo: MPI parallelization;");
}
}
}
this.BlockIndices = _BlockIndices.Select(bi => bi.ToArray()).ToArray();
}
// test DOF blocks
// ===============
{
int L = M.RowPartitioning.LocalLength;
int i0 = M.RowPartitioning.i0;
// ensure that each cell is used exactly once, among all blocks
BitArray test = new BitArray(L);
foreach (var bi in BlockIndices)
{
bi.ForEach(delegate(int i) {
if (i < L)
{
Debug.Assert(test[i] == false || this.overlap > 0);
test[i] = true;
}
});
}
for (int i = 0; i < L; i++)
{
Debug.Assert(test[i] == true);
}
}
// create solvers
// ==============
{
blockSolvers = new ISparseSolver[NoParts];
for (int iPart = 0; iPart < NoParts; iPart++)
{
int[] bi = BlockIndices[iPart];
int Bsz;
if (MgMap.MinimalLength == MgMap.MaximalLength)
{
Bsz = MgMap.MaximalLength;
}
else
{
Bsz = 1;
}
if (M.RowPartitioning.MpiSize > 1)
{
int i0Proc = M.RowPartitioning.i0;
bi = bi.CloneAs();
for (int i = 0; i < bi.Length; i++)
{
bi[i] += i0Proc;
}
}
MsrMatrix Block = new MsrMatrix(bi.Length, bi.Length, Bsz, Bsz);
M.WriteSubMatrixTo(Block, bi, default(int[]), bi, default(int[]));
//blockSolvers[iPart] = new PARDISOSolver();
//blockSolvers[iPart].CacheFactorization = true;
//blockSolvers[iPart].DefineMatrix(Block);
//blockSolvers[iPart] = new FullDirectSolver();
//blockSolvers[iPart].DefineMatrix(Block);
blockSolvers[iPart] = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
blockSolvers[iPart].DefineMatrix(Block);
}
}
this.MtxFull = new ilPSP.LinSolvers.monkey.CPU.RefMatrix(M.ToMsrMatrix());
if (CoarseSolver != null)
{
CoarseSolver.Init(op.CoarserLevel);
}
}