public static void CellwiseSubSelection(
[Values(SelectionType.all_combined, SelectionType.degrees, SelectionType.species, SelectionType.variables)] SelectionType SType
)
{
Utils.TestInit((int)SType);
Console.WriteLine("SubSelection({0})", SType);
//Arrange --- extracts entries of matrix according to hardcoded selection
int DGdegree = 2;
int GridResolution = 4;
var mgo = Utils.CreateTestMGOperator(XDGusage.all, DGdegree, MatrixShape.full_var_spec, GridResolution);
int sampleCellA = Utils.GetIdxOfFirstBlockWith(mgo.Mapping, false); //1 species
int sampleCellB = Utils.GetIdxOfFirstBlockWith(mgo.Mapping, true); //2 species
BlockMsrMatrix compA = Utils.GetCellCompMatrix(SType, mgo, sampleCellA);
BlockMsrMatrix compB = Utils.GetCellCompMatrix(SType, mgo, sampleCellB);
int iBlock = sampleCellB + mgo.Mapping.AggGrid.CellPartitioning.i0;
int i0 = mgo.Mapping.GetBlockI0(iBlock);
var block = MultidimensionalArray.Create(mgo.Mapping.GetBlockLen(iBlock), mgo.Mapping.GetBlockLen(iBlock));
mgo.OperatorMatrix.ReadBlock(i0, i0, block);
//Arrange --- setup masking, which correspond to hardcoded
SubBlockSelector sbsA = new SubBlockSelector(mgo.Mapping);
sbsA.GetDefaultSelection(SType, sampleCellA); // single spec
BlockMask maskA = new BlockMask(sbsA, null);
SubBlockSelector sbsB = new SubBlockSelector(mgo.Mapping);
sbsB.GetDefaultSelection(SType, sampleCellB); // double spec
BlockMask maskB = new BlockMask(sbsB, null);
//Arrange --- some time measurement
Stopwatch stw = new Stopwatch();
stw.Reset();
//Act --- subblock extraction
stw.Start();
var blocksA = maskA.GetDiagonalBlocks(mgo.OperatorMatrix, false, false);
var blocksB = maskB.GetDiagonalBlocks(mgo.OperatorMatrix, false, false);
stw.Stop();
//Assert ---
Assert.IsTrue(blocksA.Length == 1);
Assert.IsTrue(blocksB.Length == 1);
Assert.IsTrue(compA.RowPartitioning.LocalLength == blocksA[0].GetLength(0));
Assert.IsTrue(compB.RowPartitioning.LocalLength == blocksB[0].GetLength(0));
//Assert --- compare masking of single spec cell
Debug.Assert(compA.InfNorm() != 0.0);
compA.AccBlock(0, 0, -1.0, blocksA[0]);
Assert.IsTrue(compA.InfNorm() == 0.0);
//Assert --- compare masking of double spec cell
Debug.Assert(compB.InfNorm() != 0.0);
compB.AccBlock(0, 0, -1.0, blocksB[0]);
Assert.IsTrue(compB.InfNorm() == 0.0, String.Format("proc{0}: not fulfilled at block {1}", mgo.Mapping.MpiRank, sampleCellB));
}
void DelComputeOperatorMatrix(BlockMsrMatrix OpMtx, double[] OpAffine, UnsetteledCoordinateMapping Mapping, DGField[] CurrentState, Dictionary <SpeciesId, MultidimensionalArray> AgglomeratedCellLengthScales, double phystime)
{
DGField[] Params = null;
if (this.Control.Eq == Equation.ScalarTransport)
{
Params = this.V.ToArray();
}
else if (this.Control.Eq == Equation.HeatEq)
{
Params = null;
}
else if (this.Control.Eq == Equation.Burgers)
{
Params = CurrentState;
}
else
{
throw new NotImplementedException();
}
// compute operator
Debug.Assert(OpMtx.InfNorm() == 0.0);
Debug.Assert(OpAffine.L2Norm() == 0.0);
Operator.ComputeMatrixEx(this.LsTrk,
Mapping, Params, Mapping,
OpMtx, OpAffine, false, phystime, true,
AgglomeratedCellLengthScales, null, null,
AgglomeratedCellLengthScales.Keys.ToArray());
}
static void PrlgAndRestMtxTestRec(int p, MultigridMapping[] MgMapSeq)
{
var currentLevelMap = MgMapSeq.First();
var AggBasis = currentLevelMap.AggBasis[0];
// extract Restriciton and Prolongation Operators
BlockMsrMatrix RestOp = new BlockMsrMatrix(MgMapSeq.First(), MgMapSeq.First().ProblemMapping);
AggBasis.GetRestrictionMatrix(RestOp, MgMapSeq.First(), 0);
BlockMsrMatrix PrlgOp = RestOp.Transpose();
// restriction onto level itself
BlockMsrMatrix ShldBeEye = BlockMsrMatrix.Multiply(RestOp, PrlgOp);
ShldBeEye.AccEyeSp(-1.0);
double ErrNorm = ShldBeEye.InfNorm();
Console.WriteLine("Id norm {0} \t (level {1})", ErrNorm, currentLevelMap.AggGrid.MgLevel);
Assert.Less(ErrNorm, 1.0e-8);
if (MgMapSeq.Length > 1)
{
PrlgAndRestMtxTestRec(p, MgMapSeq.Skip(1).ToArray());
}
}
protected override double RunSolverOneStep(int TimestepNo, double phystime, double dt)
{
//phystime = 1.8;
LsUpdate(phystime);
// operator-matrix assemblieren
OperatorMatrix = new BlockMsrMatrix(ProblemMapping);
AltOperatorMatrix = new MsrMatrix(ProblemMapping);
double[] Affine = new double[OperatorMatrix.RowPartitioning.LocalLength];
MultiphaseCellAgglomerator Agg;
// Agglomerator setup
//Agg = new MultiphaseCellAgglomerator(new CutCellMetrics(MomentFittingVariant, m_quadOrder, LsTrk, LsTrk.GetSpeciesId("B")), this.THRESHOLD, false);
Agg = LsTrk.GetAgglomerator(new SpeciesId[] { LsTrk.GetSpeciesId("B") }, m_quadOrder, __AgglomerationTreshold: this.THRESHOLD);
Console.WriteLine("Inter-Process agglomeration? " + Agg.GetAgglomerator(LsTrk.GetSpeciesId("B")).AggInfo.InterProcessAgglomeration);
// operator matrix assembly
//Op.ComputeMatrixEx(LsTrk,
// ProblemMapping, null, ProblemMapping,
// OperatorMatrix, Affine, false, 0.0, true,
// Agg.CellLengthScales, null, null,
// LsTrk.SpeciesIdS.ToArray());
XSpatialOperatorMk2.XEvaluatorLinear mtxBuilder = Op.GetMatrixBuilder(base.LsTrk, ProblemMapping, null, ProblemMapping, LsTrk.SpeciesIdS.ToArray());
mtxBuilder.time = 0.0;
mtxBuilder.ComputeMatrix(OperatorMatrix, Affine);
Agg.ManipulateMatrixAndRHS(OperatorMatrix, Affine, this.ProblemMapping, this.ProblemMapping);
//Op.ComputeMatrixEx(LsTrk,
// ProblemMapping, null, ProblemMapping,
// AltOperatorMatrix, Affine, false, 0.0, true,
// Agg.CellLengthScales, null, null,
// LsTrk.SpeciesIdS.ToArray());
mtxBuilder.ComputeMatrix(AltOperatorMatrix, Affine);
Agg.ManipulateMatrixAndRHS(AltOperatorMatrix, Affine, this.ProblemMapping, this.ProblemMapping);
int nnz = this.OperatorMatrix.GetTotalNoOfNonZeros();
Console.WriteLine("Number of non-zeros in matrix: " + nnz);
int nnz2 = this.AltOperatorMatrix.GetTotalNoOfNonZeros();
Assert.IsTrue(nnz == nnz2, "Number of non-zeros in matrix different for " + OperatorMatrix.GetType() + " and " + AltOperatorMatrix.GetType());
Console.WriteLine("Number of non-zeros in matrix (reference): " + nnz2);
MsrMatrix Comp = AltOperatorMatrix.CloneAs();
Comp.Acc(-1.0, OperatorMatrix);
double CompErr = Comp.InfNorm();
double Denom = Math.Max(AltOperatorMatrix.InfNorm(), OperatorMatrix.InfNorm());
double CompErrRel = Denom > Math.Sqrt(double.Epsilon) ? CompErr / Denom : CompErr;
Console.WriteLine("Comparison: " + CompErrRel);
Assert.LessOrEqual(CompErrRel, 1.0e-7, "Huge difference between MsrMatrix and BlockMsrMatrix.");
base.TerminationKey = true;
return(0.0);
}
/// <summary>
/// computes <see cref="LaplaceMtx"/> and <see cref="LaplaceAffine"/>
/// </summary>
private void UpdateMatrices() {
using (var tr = new FuncTrace()) {
// time measurement for matrix assembly
Stopwatch stw = new Stopwatch();
stw.Start();
// console
Console.WriteLine("creating sparse system for {0} DOF's ...", T.Mapping.Ntotal);
// quadrature domain
var volQrSch = new CellQuadratureScheme(true, CellMask.GetFullMask(this.GridData, MaskType.Geometrical));
var edgQrSch = new EdgeQuadratureScheme(true, EdgeMask.GetFullMask(this.GridData, MaskType.Geometrical));
#if DEBUG
// in DEBUG mode, we compare 'MsrMatrix' (old, reference implementation) and 'BlockMsrMatrix' (new standard)
var RefLaplaceMtx = new MsrMatrix(T.Mapping);
#endif
using (new BlockTrace("SipMatrixAssembly", tr)) {
LaplaceMtx = new BlockMsrMatrix(T.Mapping);
LaplaceAffine = new double[T.Mapping.LocalLength];
LapaceIp.ComputeMatrixEx(T.Mapping, null, T.Mapping,
LaplaceMtx, LaplaceAffine,
volQuadScheme: volQrSch, edgeQuadScheme: edgQrSch);
}
#if DEBUG
LaplaceAffine.ClearEntries();
LapaceIp.ComputeMatrixEx(T.Mapping, null, T.Mapping,
RefLaplaceMtx, LaplaceAffine,
volQuadScheme: volQrSch, edgeQuadScheme: edgQrSch);
MsrMatrix ErrMtx = RefLaplaceMtx.CloneAs();
ErrMtx.Acc(-1.0, LaplaceMtx);
double err = ErrMtx.InfNorm();
double infNrm = LaplaceMtx.InfNorm();
Console.WriteLine("Matrix comparison error: " + err + ", matrix norm is: " + infNrm);
Assert.Less(err, infNrm * 1e-10, "MsrMatrix2 comparison failed.");
#endif
stw.Stop();
Console.WriteLine("done {0} sec.", stw.Elapsed.TotalSeconds);
//var JB = LapaceIp.GetFDJacobianBuilder(T.Mapping.Fields, null, T.Mapping, edgQrSch, volQrSch);
//var JacobiMtx = new BlockMsrMatrix(T.Mapping);
//var JacobiAffine = new double[T.Mapping.LocalLength];
//JB.ComputeMatrix(JacobiMtx, JacobiAffine);
//double L2ErrAffine = GenericBlas.L2Dist(JacobiAffine, LaplaceAffine);
//var ErrMtx2 = LaplaceMtx.CloneAs();
//ErrMtx2.Acc(-1.0, JacobiMtx);
//double LinfErrMtx2 = ErrMtx2.InfNorm();
//JacobiMtx.SaveToTextFileSparse("D:\\tmp\\Jac.txt");
//LaplaceMtx.SaveToTextFileSparse("D:\\tmp\\Lap.txt");
//Console.WriteLine("FD Jacobi Mtx: {0:e14}, Affine: {1:e14}", LinfErrMtx2, L2ErrAffine);
}
}
static void RestictionMatrixTestRec(int p, IEnumerable <MultigridMapping> MgMapSeq)
{
var currentLevelMap = MgMapSeq.First();
AggregationGridBasis AggBasis = currentLevelMap.AggBasis[0];
var map = new UnsetteledCoordinateMapping(new Basis(grid, p));
Random rnd = new Random();
double[] OrigVec = map.LocalLength.ForLoop(i => rnd.NextDouble());
double[] RestVec = new double[AggBasis.LocalDim];
double[] PrlgVec = new double[OrigVec.Length];
double[] RestVec2 = new double[RestVec.Length];
double[] PrlgVec2 = new double[OrigVec.Length];
AggBasis.RestictFromFullGrid(OrigVec, RestVec);
AggBasis.ProlongateToFullGrid(PrlgVec, RestVec);
BlockMsrMatrix RestOp = new BlockMsrMatrix(MgMapSeq.First(), MgMapSeq.First().ProblemMapping);
AggBasis.GetRestrictionMatrix(RestOp, MgMapSeq.First(), 0);
RestOp.SpMV(1.0, OrigVec, 0.0, RestVec2);
BlockMsrMatrix PrlgOp = RestOp.Transpose();
PrlgOp.SpMV(1.0, RestVec2, 0.0, PrlgVec2);
double RestErrNorm = GenericBlas.L2Dist(RestVec2, RestVec);
double PrlgErrNorm = GenericBlas.L2Dist(PrlgVec2, PrlgVec);
double LostInfNorm = GenericBlas.L2Dist(OrigVec, PrlgVec2);
//Console.WriteLine("Rest. matrix test: {0}, Prolong. matrix test {1}, Lost info {2}", RestErrNorm, PrlgErrNorm, LostInfNorm);
Assert.IsTrue(RestErrNorm < 1.0e-10);
Assert.IsTrue(PrlgErrNorm < 1.0e-10);
// restriction onto level itself
BlockMsrMatrix RestMtx = currentLevelMap.FromOtherLevelMatrix(currentLevelMap);
BlockMsrMatrix ShldBeEye = BlockMsrMatrix.Multiply(RestMtx, RestMtx.Transpose());
ShldBeEye.AccEyeSp(-1.0);
double errNorm = ShldBeEye.InfNorm();
Console.WriteLine("Id norm {0} \t (level {1})", errNorm, currentLevelMap.AggGrid.MgLevel);
Assert.IsTrue(errNorm < 1.0e-8);
// recursion
if (MgMapSeq.Count() > 1)
{
RestictionMatrixTestRec(p, MgMapSeq.Skip(1));
}
}
private void AssembleMatrix(double dt, out BlockMsrMatrix SystemMatrix, out double[] SystemAffine)
{
// Init Matrix and Affine Part
SystemMatrix = new BlockMsrMatrix(Mapping);
SystemAffine = new double[Mapping.LocalLength];
// choose TimeStepping-Scheme, based on what has been pushed to the stack,yet
int Smax = TSCchain[0].S;
Debug.Assert(Smax == TSCchain.Length);
Tsc = TSCchain[Smax - PopulatedStackDepth];
UpdateOperatorMatrix();
//Implicit Part of RHS
SystemMatrix.Acc(Tsc.theta1, Stack_OpMatrix[1]);
SystemAffine.AccV(Tsc.theta1, Stack_OpAffine[1]);
//Implicit Part of LHS
SystemMatrix.AccEyeSp(1 / dt);
// Explicit part of RHS
Stack_OpMatrix[0].SpMV(-Tsc.theta0, CurrentState, 1.0, SystemAffine);
SystemAffine.AccV(-Tsc.theta0, Stack_OpAffine[0]);
//Explicit parts of LHS
for (int i = 0; i < Tsc.beta.Length; i++)
{
SystemAffine.AccV(Tsc.beta[i] * 1 / dt, Stack_u[i]);
}
Debug.Assert(SystemMatrix.InfNorm() > 0);
Debug.Assert(SystemAffine.L2Norm() > 0);
if (subGrid != null)
{
int[] SubVecIdx = Mapping.GetSubvectorIndices(subGrid, true, new int[] { 0 });
int L = SubVecIdx.Length;
for (int i = 0; i < L; i++)
{
SystemMatrix.ClearRow(SubVecIdx[i]);
SystemMatrix[SubVecIdx[i], SubVecIdx[i]] = 1;
SystemAffine[SubVecIdx[i]] = 0;
}
}
}
public static void SubSelection(
[Values(SelectionType.all_combined, SelectionType.degrees, SelectionType.species, SelectionType.variables)] SelectionType SType
)
{
Utils.TestInit((int)SType);
Console.WriteLine("SubSelection({0})", SType);
//Arrange --- extracts entries of matrix according to hardcoded selection
int DGdegree = 2;
int GridResolution = 4;
var mgo = Utils.CreateTestMGOperator(XDGusage.all, DGdegree, MatrixShape.full_var_spec, GridResolution);
int sampleCellA = Utils.GetIdxOfFirstBlockWith(mgo.Mapping, false); //1 species
int sampleCellB = Utils.GetIdxOfFirstBlockWith(mgo.Mapping, true); //2 species
BlockMsrMatrix compA = Utils.GetCellCompMatrix(SType, mgo, sampleCellA);
BlockMsrMatrix compB = Utils.GetCellCompMatrix(SType, mgo, sampleCellB);
//Arrange --- setup masking, which correspond to hardcoded
SubBlockSelector sbsA = new SubBlockSelector(mgo.Mapping);
sbsA.GetDefaultSelection(SType, sampleCellA); // single spec
BlockMask maskA = new BlockMask(sbsA, null);
SubBlockSelector sbsB = new SubBlockSelector(mgo.Mapping);
sbsB.GetDefaultSelection(SType, sampleCellB); // double spec
BlockMask maskB = new BlockMask(sbsB, null);
//Arrange --- stop the watch
Stopwatch stw = new Stopwatch();
stw.Reset();
//Act --- get subblocks
stw.Start();
BlockMsrMatrix subA = maskA.GetSubBlockMatrix(mgo.OperatorMatrix);
BlockMsrMatrix subB = maskB.GetSubBlockMatrix(mgo.OperatorMatrix);
stw.Stop();
//Assert --- compare masking of single spec cell
Debug.Assert(compA.InfNorm() != 0.0);
subA.Acc(-1.0, compA);
Assert.IsTrue(subA.InfNorm() == 0.0);
//Assert --- compare masking of double spec cell
Debug.Assert(compB.InfNorm() != 0.0);
subB.Acc(-1.0, compB);
Assert.IsTrue(subB.InfNorm() == 0.0);
}
protected override void CreateEquationsAndSolvers(GridUpdateDataVaultBase L)
{
AssembleMatrix(this.Control.MU_A, this.Control.MU_B, out Op_Matrix, out Op_Affine, out Op_Agglomeration, out Op_mass);
Console.WriteLine("Matrix norm: {0}", Op_Matrix.InfNorm());
Console.WriteLine("Symm. diff: {0}", Op_Matrix.SymmetryDeviation());
}
public void Init(MultigridOperator op)
{
using (new FuncTrace()) {
if (m_MgOp != null)
{
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// someone is trying to re-use this solver: see if the settings permit that
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if (op.LevelIndex != m_MgOp.LevelIndex)
{
throw new ArgumentException("Re-use on different level not possible.");
}
if (!this.MtxFull._RowPartitioning.EqualsPartition(op.OperatorMatrix._RowPartitioning))
{
throw new ArgumentException("Matrix has changed, unable to re-use");
}
if (!this.MtxFull._ColPartitioning.EqualsPartition(op.OperatorMatrix._ColPartitioning))
{
throw new ArgumentException("Matrix has changed, unable to re-use");
}
#if DEBUG
if (!object.ReferenceEquals(this.MtxFull, op.OperatorMatrix))
{
BlockMsrMatrix Check = this.MtxFull.CloneAs();
Check.Acc(-1.0, op.OperatorMatrix);
if (Check.InfNorm() != 0.0)
{
throw new ArgumentException("Matrix has changed, unable to re-use");
}
}
#endif
if (this.m_BlockingStrategy.GetNoOfBlocks(op) != this.blockSolvers.Count())
{
throw new ArgumentException("Blocking, unable to re-use");
}
return;
}
var Mop = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_MgOp = op;
int myMpiRank = MgMap.MpiRank;
int myMpisize = MgMap.MpiSize;
if (!Mop.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!Mop.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
var ag = MgMap.AggGrid;
int JComp = ag.iLogicalCells.NoOfLocalUpdatedCells;
int JGhost = ag.iLogicalCells.NoOfExternalCells;
#if DEBUG
ilPSP.Connectors.Matlab.BatchmodeConnector matlab;
if (m_MatlabParalellizationCheck)
{
matlab = new ilPSP.Connectors.Matlab.BatchmodeConnector();
}
else
{
matlab = null;
}
#endif
//Mop.Clear();
//for(int i = Mop.RowPartitioning.i0; i < Mop.RowPartitioning.iE; i++) {
// Mop[i, i] = i + 1;
//}
// get cell blocks
// ===============
var _Blocks = this.m_BlockingStrategy.GetBlocking(op);
int NoOfSchwzBlocks = _Blocks.Count();
// test cell blocks
// ================
#if DEBUG
{
// ensure that each cell is used exactly once, among all blocks
bool[] test = new bool[ag.iLogicalCells.NoOfLocalUpdatedCells];
foreach (var bi in _Blocks)
{
foreach (int j in bi)
{
Debug.Assert(test[j] == false);
test[j] = true;
}
;
}
for (int i = 0; i < test.Length; i++)
{
Debug.Assert(test[i] == true);
}
}
#endif
// extend blocks according to desired overlap
// ==========================================
{
BitArray marker = new BitArray(JComp + JGhost);
if (Overlap < 0)
{
throw new ArgumentException();
}
if (Overlap > 0)
{
if (Overlap > 1 && Mop.RowPartitioning.MpiSize > 1)
{
throw new NotSupportedException("In MPI parallel runs, the maximum supported overlap for the Schwarz preconditioner is 1.");
}
foreach (List <int> bi in _Blocks) // loop over blocks...
{
marker.SetAll(false); // marks all cells which are members of the block
foreach (int jcomp in bi)
{
marker[jcomp] = true;
}
// determine overlap regions
for (int k = 0; k < Overlap; k++)
{
int Jblock = bi.Count;
for (int j = 0; j < Jblock; j++)
{
int jCell = bi[j];
int[] Neighs = ag.iLogicalCells.CellNeighbours[jCell];
foreach (int jNeigh in Neighs)
{
if (marker[jNeigh] == false)
{
// neighbor cell is not already a member of the block
// => add it.
bi.Add(jNeigh);
marker[jNeigh] = true;
}
}
}
}
bi.Sort();
}
}
BlockCells = _Blocks.Select(list => list.ToArray()).ToArray();
}
// convert cell blocks to DOF blocks
// =================================
List <int>[] BlkIdx_gI_lR; // for each Schwarz block, (global) indices in the local range
List <int>[] BlkIdx_gI_eR; // for each Schwarz block, (global) indices of external rows and columns
List <int>[] TempRowIdx_gI; // for each Schwarz block, (global) indices into the temporary matrix
List <int>[] BlkIdx_lI_eR; // for each Schwarz block, (local) indices of external rows and columns
List <int>[] LocalBlocks_i0, LocalBlocks_N; // blocking of the Schwarz-Blocks.
// for matrix 'ExternalRowsTemp': which rows of 'Mop' are required locally
List <int> ExternalRowsIndices, ExternalRows_BlockI0, ExternalRows_BlockN;
{
int Jup = MgMap.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
int Jgh = MgMap.AggGrid.iLogicalCells.NoOfExternalCells;
int LocalizedBlockCounter = 0;
BlkIdx_gI_lR = NoOfSchwzBlocks.ForLoop(iPart => new List <int>(BlockCells[iPart].Length * MgMap.MaximalLength));
BlkIdx_gI_eR = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
LocalBlocks_i0 = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
LocalBlocks_N = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
TempRowIdx_gI = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
BlkIdx_lI_eR = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
ExternalRowsIndices = new List <int>();
ExternalRows_BlockI0 = new List <int>();
ExternalRows_BlockN = new List <int>();
for (int iPart = 0; iPart < NoOfSchwzBlocks; iPart++) // loop over parts...
{
int[] bc = BlockCells[iPart];
var biI = BlkIdx_gI_lR[iPart];
var biE = BlkIdx_gI_eR[iPart];
var l1 = TempRowIdx_gI[iPart];
var l2 = BlkIdx_lI_eR[iPart];
var LBBi0 = LocalBlocks_i0[iPart];
var LBBN = LocalBlocks_N[iPart];
int Jblock = bc.Length;
int anotherCounter = 0;
for (int jblk = 0; jblk < Jblock; jblk++) // loop over cells in blocks...
{
int j = bc[jblk];
int N = MgMap.GetLength(j);
if (j < Jup)
{
// locally updated cell
int i0 = MgMap.GlobalUniqueIndex(0, j, 0);
for (int n = 0; n < N; n++)
{
biI.Add(i0 + n);
}
}
else
{
// external cell
int i0E = MgMap.GlobalUniqueIndex(0, j, 0); //
int i0L = MgMap.LocalUniqueIndex(0, j, 0); //
ExternalRows_BlockI0.Add(LocalizedBlockCounter);
ExternalRows_BlockN.Add(N);
//LEBi0.Add(LocalizedBlockCounter);
//LEBn.Add(N);
for (int n = 0; n < N; n++)
{
biE.Add(i0E + n);
ExternalRowsIndices.Add(i0E + n);
l1.Add(LocalizedBlockCounter + n);
l2.Add(i0L + n);
Debug.Assert(Mop._RowPartitioning.FindProcess(i0E + n) != myMpiRank);
}
LocalizedBlockCounter += N;
}
LBBi0.Add(anotherCounter);
LBBN.Add(N);
anotherCounter += N;
}
}
//this.BlockIndices = _LocallyStoredBlockIndices.Select(bi => bi.ToArray()).ToArray();
}
// get rows for blocks that use external cells
// ===========================================
#if DEBUG
{
if (Overlap == 0)
{
Debug.Assert(ExternalRowsIndices.Count == 0);
Debug.Assert(ExternalRows_BlockI0.Count == 0);
Debug.Assert(ExternalRows_BlockN.Count == 0);
}
foreach (var bi in BlkIdx_gI_lR)
{
foreach (int idx in bi)
{
Debug.Assert(idx >= m_MgOp.Mapping.i0);
Debug.Assert(idx < m_MgOp.Mapping.iE);
}
}
foreach (var ei in BlkIdx_gI_eR)
{
foreach (int idx in ei)
{
Debug.Assert(idx < m_MgOp.Mapping.i0 || idx >= m_MgOp.Mapping.iE);
}
}
int LL = m_MgOp.Mapping.LocalLength;
int jMax = m_MgOp.Mapping.AggGrid.iLogicalCells.NoOfCells - 1;
int LE = m_MgOp.Mapping.LocalUniqueIndex(0, jMax, 0) + m_MgOp.Mapping.GetLength(jMax);
foreach (var ci in BlkIdx_lI_eR)
{
foreach (int idx in ci)
{
Debug.Assert(idx >= LL);
Debug.Assert(idx < LE);
}
}
if (m_MatlabParalellizationCheck)
{
int globalBlockCounter = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
if (rankCounter == myMpiRank)
{
Debug.Assert(rank_NoBlks == NoOfSchwzBlocks);
}
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
double[] vec;
if (rankCounter == myMpiRank)
{
vec = ArrayTools.Cat(BlkIdx_gI_lR[iBlock], BlkIdx_gI_eR[iBlock]).Select(ii => ((double)(ii + 1))).ToArray();
}
else
{
vec = new double[0];
}
matlab.PutVector(vec, string.Format("BlockIdx{0}", globalBlockCounter));
globalBlockCounter++;
csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD);
}
}
}
}
#endif
BlockMsrMatrix ExternalRowsTemp;
if (myMpisize > 1 && Overlap > 0)
{
//int NoOfLocalRows = _ExternalBlockIndices.Sum(L => L.Count);
BlockPartitioning PermRow = new BlockPartitioning(ExternalRowsIndices.Count, ExternalRows_BlockI0, ExternalRows_BlockN, Mop.MPI_Comm, i0isLocal: true);
// Remark: we use a permutation matrix for MPI-exchange of rows
BlockMsrMatrix Perm = new BlockMsrMatrix(PermRow, Mop._RowPartitioning);
for (int iRow = 0; iRow < ExternalRowsIndices.Count; iRow++)
{
Debug.Assert(Mop._RowPartitioning.IsInLocalRange(ExternalRowsIndices[iRow]) == false);
Perm[iRow + PermRow.i0, ExternalRowsIndices[iRow]] = 1;
}
ExternalRowsTemp = BlockMsrMatrix.Multiply(Perm, Mop);
#if DEBUG
if (m_MatlabParalellizationCheck)
{
matlab.PutSparseMatrix(Perm, "Perm");
matlab.PutSparseMatrix(ExternalRowsTemp, "ExternalRowsTemp");
}
#endif
}
else
{
ExternalRowsTemp = null;
}
ExternalRowsIndices = null;
ExternalRows_BlockI0 = null;
ExternalRows_BlockN = null;
// create solvers
// ==============
{
blockSolvers = new ISparseSolver[NoOfSchwzBlocks];
#if DEBUG
List <BlockMsrMatrix> Blocks = new List <BlockMsrMatrix>();
#endif
for (int iPart = 0; iPart < NoOfSchwzBlocks; iPart++)
{
var bi = BlkIdx_gI_lR[iPart];
int Bsz;
if (MgMap.MinimalLength == MgMap.MaximalLength)
{
Bsz = MgMap.MaximalLength;
}
else
{
Bsz = 1;
}
var l1 = TempRowIdx_gI[iPart];
//if (M.RowPartitioning.MpiSize > 1) {
// int i0Proc = M.RowPartitioning.i0;
// bi = bi.CloneAs();
// for (int i = 0; i < bi.Length; i++) {
// bi[i] += i0Proc;
// }
//}
BlockPartitioning localBlocking = new BlockPartitioning(bi.Count + l1.Count, LocalBlocks_i0[iPart], LocalBlocks_N[iPart], csMPI.Raw._COMM.SELF);
if (l1.Count > 0)
{
// convert the indices into 'ExternalRowsTemp' to global indices
int l1L = l1.Count;
int offset = ExternalRowsTemp._RowPartitioning.i0;
for (int i = 0; i < l1L; i++)
{
l1[i] += offset;
}
}
BlockMsrMatrix Block = new BlockMsrMatrix(localBlocking, localBlocking);// bi.Length, bi.Length, Bsz, Bsz);
Mop.WriteSubMatrixTo(Block, bi, default(int[]), bi, default(int[]));
if (l1.Count > 0)
{
int offset = bi.Count;
int[] targRows = l1.Count.ForLoop(i => i + offset);
var biE = BlkIdx_gI_eR[iPart];
int[] extTargCols = biE.Count.ForLoop(i => i + offset);
Mop.AccSubMatrixTo(1.0, Block, bi, default(int[]), new int[0], default(int[]), biE, extTargCols);
ExternalRowsTemp.AccSubMatrixTo(1.0, Block, l1, targRows, bi, default(int[]), biE, extTargCols);
}
#if DEBUG
if (m_MatlabParalellizationCheck)
{
Blocks.Add(Block);
}
#endif
blockSolvers[iPart] = new PARDISOSolver()
{
CacheFactorization = true,
UseDoublePrecision = false
};
//blockSolvers[iPart] = new FullDirectSolver();
//blockSolvers[iPart] = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
blockSolvers[iPart].DefineMatrix(Block);
}
#if DEBUG
if (m_MatlabParalellizationCheck)
{
int globalBlockCounter = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
BlockMsrMatrix Block;
if (rankCounter == myMpiRank)
{
Block = Blocks[iBlock];
}
else
{
Block = null;
}
matlab.PutSparseMatrix(Block, string.Format("Block{0}", globalBlockCounter));
globalBlockCounter++;
csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD);
}
}
}
#endif
}
// Record required indices
// =======================
{
this.BlockIndices_Local = new int[NoOfSchwzBlocks][];
this.BlockIndices_External = new int[NoOfSchwzBlocks][];
int LocalI0 = MgMap.i0;
int LocalLength = MgMap.LocalLength;
for (int iBlock = 0; iBlock < NoOfSchwzBlocks; iBlock++)
{
var _bi = BlkIdx_gI_lR[iBlock];
int L = _bi.Count;
int[] bil = new int[L];
this.BlockIndices_Local[iBlock] = bil;
for (int l = 0; l < L; l++)
{
bil[l] = _bi[l] - LocalI0;
Debug.Assert(bil[l] >= 0);
Debug.Assert(bil[l] < MgMap.LocalLength);
}
var _biE = BlkIdx_lI_eR[iBlock];
if (_biE.Count > 0)
{
this.BlockIndices_External[iBlock] = _biE.ToArray();
}
}
}
this.MtxFull = Mop;
if (CoarseSolver != null)
{
CoarseSolver.Init(op.CoarserLevel);
}
// Debug & Test-Code
// =================
#if DEBUG
if (m_MatlabParalellizationCheck)
{
Console.WriteLine("Matlab dir: " + matlab.WorkingDirectory);
matlab.PutSparseMatrix(Mop, "Full");
int GlobalNoOfBlocks = NoOfSchwzBlocks.MPISum();
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
matlab.Cmd("BlockErr({0} + 1, 1) = norm( Block{0} - Full( BlockIdx{0}, BlockIdx{0} ), inf );", iGlbBlock);
}
Random rnd = new Random(myMpiRank);
double[] testRHS = new double[MgMap.LocalLength];
for (int i = 0; i < testRHS.Length; i++)
{
testRHS[i] = rnd.NextDouble();
}
matlab.PutVector(testRHS, "testRHS");
MPIexchange <double[]> ResExchange = new MPIexchange <double[]>(MgMap, testRHS);
ResExchange.TransceiveStartImReturn();
ResExchange.TransceiveFinish(0.0);
int offset = MgMap.LocalLength;
int g = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
double[] SubVec;
if (rankCounter == myMpiRank)
{
int LL = this.BlockIndices_Local[iBlock].Length;
int LE;
if (this.BlockIndices_External[iBlock] != null)
{
LE = this.BlockIndices_External[iBlock].Length;
}
else
{
LE = 0;
}
int L = LL + LE;
SubVec = new double[L];
for (int i = 0; i < LL; i++)
{
SubVec[i] = testRHS[this.BlockIndices_Local[iBlock][i]];
}
if (LE > 0)
{
for (int i = 0; i < LE; i++)
{
SubVec[i + LL] = ResExchange.Vector_Ext[this.BlockIndices_External[iBlock][i] - offset];
}
}
}
else
{
SubVec = new double[0];
}
matlab.PutVector(SubVec, "SubVec" + g);
g++;
}
}
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
matlab.Cmd("RhsErr({0} + 1, 1) = norm( SubVec{0} - testRHS( BlockIdx{0} ), inf );", iGlbBlock);
}
double[] testX = new double[testRHS.Length];
MPIexchangeInverse <double[]> XExchange = new MPIexchangeInverse <double[]>(MgMap, testX);
g = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
if (rankCounter == myMpiRank)
{
int LL = this.BlockIndices_Local[iBlock].Length;
int LE;
if (this.BlockIndices_External[iBlock] != null)
{
LE = this.BlockIndices_External[iBlock].Length;
}
else
{
LE = 0;
}
int L = LL + LE;
for (int i = 0; i < LL; i++)
{
testX[this.BlockIndices_Local[iBlock][i]] += (g + 1);
}
if (LE > 0)
{
for (int i = 0; i < LE; i++)
{
XExchange.Vector_Ext[this.BlockIndices_External[iBlock][i] - offset] += (g + 1);
}
}
}
else
{
//nop
}
g++;
}
}
XExchange.TransceiveStartImReturn();
XExchange.TransceiveFinish(1.0);
matlab.Cmd("testXref = zeros({0},1);", MgMap.TotalLength);
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
matlab.Cmd("testXref(BlockIdx{0},1) = testXref(BlockIdx{0},1) + ({0} + 1);", iGlbBlock);
}
matlab.PutVector(testX, "testX");
matlab.Cmd("testXErr = norm(testX - testXref, inf);");
MultidimensionalArray BlockErr = MultidimensionalArray.Create(GlobalNoOfBlocks, 1);
MultidimensionalArray RhsErr = MultidimensionalArray.Create(GlobalNoOfBlocks, 1);
MultidimensionalArray testXErr = MultidimensionalArray.Create(1, 1);
matlab.GetMatrix(BlockErr, "BlockErr");
matlab.GetMatrix(RhsErr, "RhsErr");
matlab.GetMatrix(testXErr, "testXErr");
matlab.Execute();
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
Console.WriteLine("Block #{0} Error (external? ) " + BlockErr[iGlbBlock, 0], iGlbBlock);
Console.WriteLine("RHS #{0} Error " + RhsErr[iGlbBlock, 0], iGlbBlock);
Debug.Assert(BlockErr[iGlbBlock, 0] == 0);
Debug.Assert(RhsErr[iGlbBlock, 0] == 0);
}
Console.WriteLine("X Error " + testXErr[0, 0]);
Debug.Assert(testXErr[0, 0] == 0.0);
matlab.Dispose();
}
#endif
}
}
public static void SubMatrixTest(
[Values(XDGusage.none, XDGusage.mixed1, XDGusage.mixed2, XDGusage.all)] XDGusage UseXdg,
[Values(1, 3)] int DGOrder,
[Values(false, true)] bool compressL1,
[Values(false, true)] bool compressL2)
{
unsafe
{
int[] Params = new int[8], ParamsGlob = new int[8];
fixed(int *pParams = Params, pParamsGlob = ParamsGlob)
{
pParams[0] = (int)UseXdg;
pParams[1] = DGOrder;
pParams[2] = compressL1 ? 1 : 0;
pParams[3] = compressL2 ? 1 : 0;
pParams[4] = -pParams[0];
pParams[5] = -pParams[1];
pParams[6] = -pParams[2];
pParams[7] = -pParams[3];
csMPI.Raw.Allreduce((IntPtr)pParams, (IntPtr)pParamsGlob, 8, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MIN, csMPI.Raw._COMM.WORLD);
}
int[] ParamsMin = ParamsGlob.GetSubVector(0, 4);
int[] ParamsMax = ParamsGlob.GetSubVector(4, 4);
for (int i = 0; i < 4; i++)
{
if (Params[i] != ParamsMin[i])
{
throw new ApplicationException();
}
if (Params[i] != -ParamsMax[i])
{
throw new ApplicationException();
}
}
Console.WriteLine("SubMatrixTest({0},{1},{2},{3})", UseXdg, DGOrder, compressL1, compressL2);
}
using (var solver = new Matrix_MPItestMain()
{
m_UseXdg = UseXdg, m_DGorder = DGOrder
}) {
// create the test data
// ====================
BoSSS.Solution.Application.CommandLineOptions opts = null;
//opts = new BoSSS.Solution.Application.CommandLineOptions();
solver.Init(null, opts);
solver.RunSolverMode();
Stopwatch stw = new Stopwatch();
stw.Reset();
stw.Start();
BlockMsrMatrix M = solver.OperatorMatrix;
int[] Ilist1 = solver.ProblemMapping.GetSubvectorIndices(false, 0);
int[] Ilist2 = solver.ProblemMapping.GetSubvectorIndices(false, 1);
foreach (int i in Ilist1)
{
Assert.IsTrue(solver.ProblemMapping.IsInLocalRange(i));
}
foreach (int i in Ilist2)
{
Assert.IsTrue(solver.ProblemMapping.IsInLocalRange(i));
}
var Blk1 = solver.ProblemMapping.GetSubBlocking(Ilist1, csMPI.Raw._COMM.WORLD, compressL1 ? -1 : 0);
var Blk2 = solver.ProblemMapping.GetSubBlocking(Ilist2, csMPI.Raw._COMM.WORLD, compressL2 ? -1 : 0);
int[] Tlist1 = compressL1 ? default(int[]) : Blk1.GetOccupiedIndicesList();
int[] Tlist2 = compressL2 ? default(int[]) : Blk2.GetOccupiedIndicesList();
if (Tlist1 != null)
{
Assert.AreEqual(Tlist1.Length, Ilist1.Length);
foreach (int i in Tlist1)
{
Assert.IsTrue(Blk1.IsInLocalRange(i));
}
}
if (Tlist2 != null)
{
Assert.AreEqual(Tlist2.Length, Ilist2.Length);
foreach (int i in Tlist2)
{
Assert.IsTrue(Blk2.IsInLocalRange(i));
}
}
BlockMsrMatrix M11 = new BlockMsrMatrix(Blk1, Blk1);
BlockMsrMatrix M12 = new BlockMsrMatrix(Blk1, Blk2);
BlockMsrMatrix M21 = new BlockMsrMatrix(Blk2, Blk1);
BlockMsrMatrix M22 = new BlockMsrMatrix(Blk2, Blk2);
M.AccSubMatrixTo(1.0, M11, Ilist1, Tlist1, Ilist1, Tlist1);
M.AccSubMatrixTo(1.0, M12, Ilist1, Tlist1, Ilist2, Tlist2);
M.AccSubMatrixTo(1.0, M21, Ilist2, Tlist2, Ilist1, Tlist1);
M.AccSubMatrixTo(1.0, M22, Ilist2, Tlist2, Ilist2, Tlist2);
BlockMsrMatrix restored_M = new BlockMsrMatrix(M._RowPartitioning, M._ColPartitioning);
int[] Idx1 = compressL1 ? Blk1.LocalLength.ForLoop(i => i + Blk1.i0) : Tlist1;
int[] Idx2 = compressL2 ? Blk2.LocalLength.ForLoop(i => i + Blk2.i0) : Tlist2;
M11.AccSubMatrixTo(1.0, restored_M, Idx1, Ilist1, Idx1, Ilist1);
M12.AccSubMatrixTo(1.0, restored_M, Idx1, Ilist1, Idx2, Ilist2);
M21.AccSubMatrixTo(1.0, restored_M, Idx2, Ilist2, Idx1, Ilist1);
M22.AccSubMatrixTo(1.0, restored_M, Idx2, Ilist2, Idx2, Ilist2);
// test transpose-operator
var M_TT = M.Transpose().Transpose();
var M11_TT = M11.Transpose().Transpose();
var M12_TT = M12.Transpose().Transpose();
var M21_TT = M21.Transpose().Transpose();
var M22_TT = M22.Transpose().Transpose();
M_TT.Acc(-1.0, M);
M11_TT.Acc(-1.0, M11);
M12_TT.Acc(-1.0, M12);
M21_TT.Acc(-1.0, M21);
M22_TT.Acc(-1.0, M22);
double M_TT_norm = M_TT.InfNorm();
double M11_TT_norm = M11_TT.InfNorm();
double M12_TT_norm = M12_TT.InfNorm();
double M21_TT_norm = M21_TT.InfNorm();
double M22_TT_norm = M22_TT.InfNorm();
Assert.IsTrue(M_TT_norm == 0.0, "Transpose^2 is not identity.");
Assert.IsTrue(M11_TT_norm == 0.0, "Transpose^2 is not identity.");
Assert.IsTrue(M12_TT_norm == 0.0, "Transpose^2 is not identity.");
Assert.IsTrue(M21_TT_norm == 0.0, "Transpose^2 is not identity.");
Assert.IsTrue(M22_TT_norm == 0.0, "Transpose^2 is not identity.");
//M.SaveToTextFileSparse(@"C:\tmp\M.txt");
//M11.SaveToTextFileSparse(@"C:\tmp\M11.txt");
//M12.SaveToTextFileSparse(@"C:\tmp\M12.txt");
//M21.SaveToTextFileSparse(@"C:\tmp\M21.txt");
//M22.SaveToTextFileSparse(@"C:\tmp\M22.txt");
//restored_M.SaveToTextFileSparse(@"C:\tmp\Mr.txt");
stw.Stop();
using (var MatlabRef = new BatchmodeConnector()) {
MatlabRef.PutVector(Ilist1.Select(i => (double)i + 1.0).ToArray(), "Ilist1");
MatlabRef.PutVector(Ilist2.Select(i => (double)i + 1.0).ToArray(), "Ilist2");
MatlabRef.PutVector(Tlist1 == null ? Ilist1.Length.ForLoop(i => (double)i + 1.0 + Blk1.i0) : Tlist1.Select(i => (double)i + 1.0).ToArray(), "Tlist1");
MatlabRef.PutVector(Tlist2 == null ? Ilist2.Length.ForLoop(i => (double)i + 1.0 + Blk2.i0) : Tlist2.Select(i => (double)i + 1.0).ToArray(), "Tlist2");
MultidimensionalArray CheckRes = MultidimensionalArray.Create(1, 4);
MatlabRef.PutSparseMatrix(M, "M");
MatlabRef.PutSparseMatrix(M11, "M11");
MatlabRef.PutSparseMatrix(M12, "M12");
MatlabRef.PutSparseMatrix(M21, "M21");
MatlabRef.PutSparseMatrix(M22, "M22");
MatlabRef.Cmd("L1 = {0};", Blk1.TotalLength);
MatlabRef.Cmd("L2 = {0};", Blk2.TotalLength);
MatlabRef.Cmd("refM11 = sparse(L1, L1);");
MatlabRef.Cmd("refM12 = sparse(L1, L2);");
MatlabRef.Cmd("refM21 = sparse(L2, L1);");
MatlabRef.Cmd("refM22 = sparse(L2, L2);");
MatlabRef.Cmd("refM11(Tlist1, Tlist1) = M(Ilist1, Ilist1);");
MatlabRef.Cmd("refM12(Tlist1, Tlist2) = M(Ilist1, Ilist2);");
MatlabRef.Cmd("refM21(Tlist2, Tlist1) = M(Ilist2, Ilist1);");
MatlabRef.Cmd("refM22(Tlist2, Tlist2) = M(Ilist2, Ilist2);");
MatlabRef.Cmd("err11 = norm(refM11 - M11, inf);");
MatlabRef.Cmd("err12 = norm(refM12 - M12, inf);");
MatlabRef.Cmd("err21 = norm(refM21 - M21, inf);");
MatlabRef.Cmd("err22 = norm(refM22 - M22, inf);");
MatlabRef.Cmd("CheckRes = [err11, err12, err21, err22];");
MatlabRef.GetMatrix(CheckRes, "CheckRes");
MatlabRef.Execute();
Console.WriteLine("Matlab check 11: " + CheckRes[0, 0]);
Console.WriteLine("Matlab check 12: " + CheckRes[0, 1]);
Console.WriteLine("Matlab check 21: " + CheckRes[0, 2]);
Console.WriteLine("Matlab check 22: " + CheckRes[0, 3]);
Assert.IsTrue(CheckRes[0, 0] == 0.0);
Assert.IsTrue(CheckRes[0, 1] == 0.0);
Assert.IsTrue(CheckRes[0, 2] == 0.0);
Assert.IsTrue(CheckRes[0, 3] == 0.0);
}
stw.Start();
restored_M.Acc(-1.0, M);
double err = restored_M.InfNorm();
Console.WriteLine("Submatrix operations error: " + err);
Assert.IsTrue(err == 0.0);
restored_M.Clear();
restored_M.Acc(1.0, M);
IMutuableMatrixEx_Extensions.Acc(restored_M, -1.0, M);
double err2 = restored_M.InfNorm();
Console.WriteLine("Submatrix operations error: " + err2);
Assert.IsTrue(err2 == 0.0);
stw.Stop();
Console.WriteLine("Time spend in matrix operations: " + stw.Elapsed.TotalSeconds + " sec.");
TotTime_MatrixOp += stw.Elapsed;
}
}
public static void MultiplyTest(
[Values(XDGusage.none, XDGusage.mixed1, XDGusage.mixed2, XDGusage.all)] XDGusage UseXdg,
[Values(1, 3)] int DGOrder,
[Values(false, true)] bool compressL1,
[Values(false, true)] bool compressL2)
{
unsafe
{
int[] Params = new int[8], ParamsGlob = new int[8];
fixed(int *pParams = Params, pParamsGlob = ParamsGlob)
{
pParams[0] = (int)UseXdg;
pParams[1] = DGOrder;
pParams[2] = compressL1 ? 1 : 0;
pParams[3] = compressL2 ? 1 : 0;
pParams[4] = -pParams[0];
pParams[5] = -pParams[1];
pParams[6] = -pParams[2];
pParams[7] = -pParams[3];
csMPI.Raw.Allreduce((IntPtr)pParams, (IntPtr)pParamsGlob, 8, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MIN, csMPI.Raw._COMM.WORLD);
}
int[] ParamsMin = ParamsGlob.GetSubVector(0, 4);
int[] ParamsMax = ParamsGlob.GetSubVector(4, 4);
for (int i = 0; i < 4; i++)
{
if (Params[i] != ParamsMin[i])
{
throw new ApplicationException();
}
if (Params[i] != -ParamsMax[i])
{
throw new ApplicationException();
}
}
Console.WriteLine("MultiplyTest({0},{1},{2},{3})", UseXdg, DGOrder, compressL1, compressL2);
}
using (var solver = new Matrix_MPItestMain()
{
m_UseXdg = UseXdg, m_DGorder = DGOrder
}) {
// create the test data
// ====================
solver.Init(null);
solver.RunSolverMode();
Stopwatch stw = new Stopwatch();
stw.Reset();
stw.Start();
BlockMsrMatrix M = solver.OperatorMatrix;
int[] Ilist1 = solver.ProblemMapping.GetSubvectorIndices(false, 0);
int[] Ilist2 = solver.ProblemMapping.GetSubvectorIndices(false, 1);
foreach (int i in Ilist1)
{
Assert.IsTrue(solver.ProblemMapping.IsInLocalRange(i));
}
foreach (int i in Ilist2)
{
Assert.IsTrue(solver.ProblemMapping.IsInLocalRange(i));
}
var Blk1 = solver.ProblemMapping.GetSubBlocking(Ilist1, csMPI.Raw._COMM.WORLD, compressL1 ? -1 : 0);
var Blk2 = solver.ProblemMapping.GetSubBlocking(Ilist2, csMPI.Raw._COMM.WORLD, compressL2 ? -1 : 0);
int[] Tlist1 = compressL1 ? default(int[]) : Blk1.GetOccupiedIndicesList();
int[] Tlist2 = compressL2 ? default(int[]) : Blk2.GetOccupiedIndicesList();
if (Tlist1 != null)
{
Assert.AreEqual(Tlist1.Length, Ilist1.Length);
foreach (int i in Tlist1)
{
Assert.IsTrue(Blk1.IsInLocalRange(i));
}
}
if (Tlist2 != null)
{
Assert.AreEqual(Tlist2.Length, Ilist2.Length);
foreach (int i in Tlist2)
{
Assert.IsTrue(Blk2.IsInLocalRange(i));
}
}
BlockMsrMatrix M11 = new BlockMsrMatrix(Blk1, Blk1);
BlockMsrMatrix M12 = new BlockMsrMatrix(Blk1, Blk2);
BlockMsrMatrix M21 = new BlockMsrMatrix(Blk2, Blk1);
BlockMsrMatrix M22 = new BlockMsrMatrix(Blk2, Blk2);
M.AccSubMatrixTo(1.0, M11, Ilist1, Tlist1, Ilist1, Tlist1);
M.AccSubMatrixTo(1.0, M12, Ilist1, Tlist1, Ilist2, Tlist2);
M.AccSubMatrixTo(1.0, M21, Ilist2, Tlist2, Ilist1, Tlist1);
M.AccSubMatrixTo(1.0, M22, Ilist2, Tlist2, Ilist2, Tlist2);
/*
* MultidimensionalArray CheckRes2 = MultidimensionalArray.Create(1, 4);
* using (var MatlabRef = new BatchmodeConnector()) {
*
* MatlabRef.PutVector(Ilist1.Select(i => (double)i + 1.0).ToArray(), "Ilist1");
* MatlabRef.PutVector(Ilist2.Select(i => (double)i + 1.0).ToArray(), "Ilist2");
* MatlabRef.PutVector(Tlist1 == null ? Ilist1.Length.ForLoop(i => (double)i + 1.0 + Blk1.i0) : Tlist1.Select(i => (double)i + 1.0).ToArray(), "Tlist1");
* MatlabRef.PutVector(Tlist2 == null ? Ilist2.Length.ForLoop(i => (double)i + 1.0 + Blk2.i0) : Tlist2.Select(i => (double)i + 1.0).ToArray(), "Tlist2");
*
* MatlabRef.PutSparseMatrix(solver.AltOperatorMatrix, "M");
*
*
* MatlabRef.Cmd("L1 = {0};", Blk1.TotalLength);
* MatlabRef.Cmd("L2 = {0};", Blk2.TotalLength);
* //MatlabRef.Cmd("refM11 = sparse(L1, L1);");
* //MatlabRef.Cmd("refM12 = sparse(L1, L2);");
* MatlabRef.Cmd("refM21 = sparse(L2, L1);");
* //MatlabRef.Cmd("refM22 = sparse(L2, L2);");
*
* //MatlabRef.Cmd("refM11(Tlist1, Tlist1) = M(Ilist1, Ilist1);");
* //MatlabRef.Cmd("refM12(Tlist1, Tlist2) = M(Ilist1, Ilist2);");
* MatlabRef.Cmd("refM21(Tlist2, Tlist1) = M(Ilist2, Ilist1);");
* //MatlabRef.Cmd("refM22(Tlist2, Tlist2) = M(Ilist2, Ilist2);");
*
* //MatlabRef.Cmd("err11 = norm(refM11 - M11, inf);");
* //MatlabRef.Cmd("err12 = norm(refM12 - M12, inf);");
* //MatlabRef.Cmd("err21 = norm(refM21 - M21, inf);");
* //MatlabRef.Cmd("err22 = norm(refM22 - M22, inf);");
*
* MatlabRef.Cmd("CheckRes = [refM21(1339, 1321), 0.0, 1.567, 0 ];");
* MatlabRef.GetMatrix(CheckRes2, "CheckRes");
*
* MatlabRef.Execute();
* }
*/
// test multipliation (later verified by matlab)
BlockMsrMatrix M11xM12 = new BlockMsrMatrix(M11._RowPartitioning, M12._ColPartitioning);
M11xM12.Acc(1.0, M12);
BlockMsrMatrix.Multiply(M11xM12, M11, M12);
BlockMsrMatrix M22xM21 = new BlockMsrMatrix(M22._RowPartitioning, M21._ColPartitioning);
BlockMsrMatrix.Multiply(M22xM21, M22, M21);
double ProdNorm = M22xM21.InfNorm();
stw.Stop();
//M.SaveToTextFileSparse(@"C:\tmp\M.txt");
//M11.SaveToTextFileSparse(@"C:\tmp\M11.txt");
//M12.SaveToTextFileSparse(@"C:\tmp\M12.txt");
//M21.SaveToTextFileSparse(@"C:\tmp\M21.txt");
//M22.SaveToTextFileSparse(@"C:\tmp\M22.txt");
//M22xM21.SaveToTextFileSparse(@"C:\tmp\M22xM21.txt");
using (var MatlabRef = new BatchmodeConnector()) {
MultidimensionalArray CheckRes = MultidimensionalArray.Create(1, 4);
MatlabRef.PutSparseMatrix(M11, "M11");
MatlabRef.PutSparseMatrix(M12, "M12");
MatlabRef.PutSparseMatrix(M21, "M21");
MatlabRef.PutSparseMatrix(M22, "M22");
MatlabRef.PutSparseMatrix(M11xM12, "M11xM12");
MatlabRef.PutSparseMatrix(M22xM21, "M22xM21");
MatlabRef.Cmd("refM11xM12 = M12 + M11*M12;");
MatlabRef.Cmd("refM22xM21 = M22*M21;");
MatlabRef.Cmd("err1112 = norm(refM11xM12 - M11xM12, inf);");
MatlabRef.Cmd("err2221 = norm(refM22xM21 - M22xM21, inf);");
MatlabRef.Cmd("CheckRes = [err1112, err2221, 0, 0];");
MatlabRef.GetMatrix(CheckRes, "CheckRes");
MatlabRef.Execute();
Console.WriteLine("Matlab check M11*M12: " + CheckRes[0, 0]);
Console.WriteLine("Matlab check M22*M21: " + CheckRes[0, 1]);
Assert.IsTrue(CheckRes[0, 0] == 0.0);
Assert.IsTrue(CheckRes[0, 1] < 1.0e-10 * ProdNorm);
//Assert.IsTrue(CheckRes[0, 2] == 0.0);
//Assert.IsTrue(CheckRes[0, 3] == 0.0);
}
Console.WriteLine("Time spend in matrix operations: " + stw.Elapsed.TotalSeconds + " sec.");
TotTime_MatrixOp += stw.Elapsed;
}
}
public void ComputeOperatorMatrix(BlockMsrMatrix OpMtx, double[] OpAffine, UnsetteledCoordinateMapping Mapping, DGField[] __CurrentState, Dictionary <SpeciesId, MultidimensionalArray> AgglomeratedCellLengthScales, double time)
{
// compute operator
Debug.Assert(OpAffine.L2Norm() == 0.0);
// all kinds of checks
if (!this.CurrentState.EqualsPartition(Mapping))
{
throw new ApplicationException("something is weired");
}
if (OpMtx != null)
{
if (!OpMtx.RowPartitioning.EqualsPartition(Mapping))
{
throw new ArgumentException("Codomain/Matrix Row mapping mismatch.");
}
if (!OpMtx.ColPartition.EqualsPartition(Mapping))
{
throw new ArgumentException("Domain/Matrix column mapping mismatch.");
}
}
if (XdgOperator != null)
{
if (OpMtx != null)
{
// +++++++++++++++++++++++++++++
// Solver requires linearization
// +++++++++++++++++++++++++++++
Debug.Assert(OpMtx.InfNorm() == 0.0);
switch (XdgOperator.LinearizationHint)
{
case LinearizationHint.AdHoc: {
this.XdgOperator.InvokeParameterUpdate(__CurrentState, this.Parameters.ToArray());
var mtxBuilder = XdgOperator.GetMatrixBuilder(LsTrk, Mapping, this.Parameters, Mapping);
mtxBuilder.time = time;
mtxBuilder.MPITtransceive = true;
mtxBuilder.ComputeMatrix(OpMtx, OpAffine);
return;
}
case LinearizationHint.FDJacobi: {
var mtxBuilder = XdgOperator.GetFDJacobianBuilder(LsTrk, __CurrentState, this.Parameters, Mapping);
mtxBuilder.time = time;
mtxBuilder.MPITtransceive = true;
mtxBuilder.ComputeMatrix(OpMtx, OpAffine);
return;
}
case LinearizationHint.GetJacobiOperator: {
var op = GetJacobiXdgOperator();
if (JacobiParameterVars == null)
{
JacobiParameterVars = op.InvokeParameterFactory(this.CurrentState);
}
op.InvokeParameterUpdate(__CurrentState, JacobiParameterVars);
var mtxBuilder = op.GetMatrixBuilder(LsTrk, Mapping, this.JacobiParameterVars, Mapping);
mtxBuilder.time = time;
mtxBuilder.MPITtransceive = true;
mtxBuilder.ComputeMatrix(OpMtx, OpAffine);
return;
}
}
}
else
{
// ++++++++++++++++++++++++
// only operator evaluation
// ++++++++++++++++++++++++
this.XdgOperator.InvokeParameterUpdate(__CurrentState, this.Parameters.ToArray());
var eval = XdgOperator.GetEvaluatorEx(__CurrentState, this.Parameters, Mapping);
eval.time = time;
eval.MPITtransceive = true;
eval.Evaluate(1.0, 0.0, OpAffine);
}
}
else if (DgOperator != null)
{
if (OpMtx != null)
{
// +++++++++++++++++++++++++++++
// Solver requires linearization
// +++++++++++++++++++++++++++++
Debug.Assert(OpMtx.InfNorm() == 0.0);
switch (DgOperator.LinearizationHint)
{
case LinearizationHint.AdHoc: {
this.DgOperator.InvokeParameterUpdate(__CurrentState, this.Parameters.ToArray());
var mtxBuilder = DgOperator.GetMatrixBuilder(Mapping, this.Parameters, Mapping);
mtxBuilder.time = time;
mtxBuilder.MPITtransceive = true;
mtxBuilder.ComputeMatrix(OpMtx, OpAffine);
return;
}
case LinearizationHint.FDJacobi: {
var mtxBuilder = DgOperator.GetFDJacobianBuilder(__CurrentState, this.Parameters, Mapping);
mtxBuilder.time = time;
mtxBuilder.MPITtransceive = true;
mtxBuilder.ComputeMatrix(OpMtx, OpAffine);
return;
}
case LinearizationHint.GetJacobiOperator: {
var op = GetJacobiDgOperator();
if (JacobiParameterVars == null)
{
JacobiParameterVars = op.InvokeParameterFactory(__CurrentState);
}
op.InvokeParameterUpdate(__CurrentState, JacobiParameterVars);
var mtxBuilder = op.GetMatrixBuilder(Mapping, this.JacobiParameterVars, Mapping);
mtxBuilder.time = time;
mtxBuilder.MPITtransceive = true;
mtxBuilder.ComputeMatrix(OpMtx, OpAffine);
return;
}
}
}
else
{
// ++++++++++++++++++++++++
// only operator evaluation
// ++++++++++++++++++++++++
this.DgOperator.InvokeParameterUpdate(__CurrentState, this.Parameters.ToArray());
var eval = DgOperator.GetEvaluatorEx(__CurrentState, this.Parameters, Mapping);
eval.time = time;
eval.MPITtransceive = true;
eval.Evaluate(1.0, 0.0, OpAffine);
}
}
else
{
throw new NotImplementedException();
}
}
protected override double RunSolverOneStep(int TimestepNo, double phystime, double dt)
{
LsUpdate(phystime);
// operator-matrix assemblieren
OperatorMatrix = new BlockMsrMatrix(MG_Mapping.ProblemMapping);
AltOperatorMatrix = new MsrMatrix(MG_Mapping.ProblemMapping);
double[] Affine = new double[OperatorMatrix.RowPartitioning.LocalLength];
MultiphaseCellAgglomerator Agg;
Agg = LsTrk.GetAgglomerator(this.LsTrk.SpeciesIdS.ToArray(), m_quadOrder, __AgglomerationTreshold: this.THRESHOLD);
XSpatialOperatorMk2.XEvaluatorLinear mtxBuilder = Op.GetMatrixBuilder(base.LsTrk, MG_Mapping.ProblemMapping, null, MG_Mapping.ProblemMapping);
mtxBuilder.time = 0.0;
mtxBuilder.ComputeMatrix(OperatorMatrix, Affine);
Agg.ManipulateMatrixAndRHS(OperatorMatrix, Affine, MG_Mapping.ProblemMapping, MG_Mapping.ProblemMapping);
foreach (var S in this.LsTrk.SpeciesNames)
{
Console.WriteLine(" Species {0}: no of agglomerated cells: {1}",
S, Agg.GetAgglomerator(this.LsTrk.GetSpeciesId(S)).AggInfo.SourceCells.NoOfItemsLocally);
}
MGOp = new MultigridOperator(XAggB, map,
OperatorMatrix,
this.massFact.GetMassMatrix(map, false),
OpConfig, null);
Debug.Assert(MGOp.OperatorMatrix != null);
Debug.Assert(MGOp.Mapping != null);
someVec = GetRHS(Affine, OperatorMatrix);
mtxBuilder.ComputeMatrix(AltOperatorMatrix, Affine);
Agg.ManipulateMatrixAndRHS(AltOperatorMatrix, Affine, MG_Mapping.ProblemMapping, MG_Mapping.ProblemMapping);
//LsTrk.GetSpeciesName(((XdgAggregationBasis)MGOp.Mapping.AggBasis[0]).UsedSpecies[1]);
//LsTrk.GetSpeciesName(((XdgAggregationBasis)MGOp.Mapping.AggBasis[0]).UsedSpecies[0]);
int nnz = this.OperatorMatrix.GetTotalNoOfNonZeros();
Console.WriteLine("Number of non-zeros in matrix: " + nnz);
int nnz2 = this.AltOperatorMatrix.GetTotalNoOfNonZeros();
Assert.IsTrue(nnz == nnz2, "Number of non-zeros in matrix different for " + OperatorMatrix.GetType() + " and " + AltOperatorMatrix.GetType());
Console.WriteLine("Number of non-zeros in matrix (reference): " + nnz2);
MsrMatrix Comp = AltOperatorMatrix.CloneAs();
Comp.Acc(-1.0, OperatorMatrix);
double CompErr = Comp.InfNorm();
double Denom = Math.Max(AltOperatorMatrix.InfNorm(), OperatorMatrix.InfNorm());
double CompErrRel = Denom > Math.Sqrt(double.Epsilon) ? CompErr / Denom : CompErr;
Console.WriteLine("Comparison: " + CompErrRel);
Assert.LessOrEqual(CompErrRel, 1.0e-7, "Huge difference between MsrMatrix and BlockMsrMatrix.");
base.TerminationKey = true;
return(0.0);
}
/// <summary>
/// %
/// </summary>
public void Solve <U, V>(U X, V B)
where U : IList <double>
where V : IList <double> //
{
using (var tr = new FuncTrace()) {
double[] Residual = this.TestSolution ? B.ToArray() : null;
string SolverName = "NotSet";
using (var solver = GetSolver(m_Mtx)) {
SolverName = solver.GetType().FullName;
//Console.Write("Direct solver run {0}, using {1} ... ", IterCnt, solver.GetType().Name);
IterCnt++;
solver.Solve(X, B);
//Console.WriteLine("done.");
}
m_ThisLevelIterations++;
if (Residual != null)
{
//Console.Write("Checking residual (run {0}) ... ", IterCnt - 1);
double RhsNorm = Residual.L2NormPow2().MPISum().Sqrt();
double MatrixInfNorm = m_Mtx.InfNorm();
m_Mtx.SpMV(-1.0, X, 1.0, Residual);
double ResidualNorm = Residual.L2NormPow2().MPISum().Sqrt();
double SolutionNorm = X.L2NormPow2().MPISum().Sqrt();
double Denom = Math.Max(MatrixInfNorm, Math.Max(RhsNorm, Math.Max(SolutionNorm, Math.Sqrt(double.Epsilon))));
double RelResidualNorm = ResidualNorm / Denom;
//Console.WriteLine("done: Abs.: {0}, Rel.: {1}", ResidualNorm, RelResidualNorm);
if (RelResidualNorm > 1.0e-10)
{
//Console.WriteLine("High residual from direct solver: abs {0}, rel {1}", ResidualNorm , ResidualNorm / SolutionNorm);
m_Mtx.SaveToTextFileSparse("Mtx.txt");
X.SaveToTextFile("X.txt");
B.SaveToTextFile("B.txt");
string ErrMsg;
using (var stw = new StringWriter()) {
stw.WriteLine("High residual from direct solver (using {0}).", SolverName);
stw.WriteLine(" L2 Norm of RHS: " + RhsNorm);
stw.WriteLine(" L2 Norm of Solution: " + SolutionNorm);
stw.WriteLine(" L2 Norm of Residual: " + ResidualNorm);
stw.WriteLine(" Relative Residual norm: " + RelResidualNorm);
stw.WriteLine(" Matrix Inf norm: " + MatrixInfNorm);
stw.WriteLine("Dumping text versions of Matrix, Solution and RHS.");
ErrMsg = stw.ToString();
}
Console.WriteLine(ErrMsg);
}
}
if (this.IterationCallback != null)
{
double[] _xl = X.ToArray();
double[] _bl = B.ToArray();
m_Mtx.SpMV(-1.0, _xl, 1.0, _bl);
this.IterationCallback(1, _xl, _bl, this.m_MultigridOp);
}
}
}
/// <summary>
/// Includes assembly of the matrix.
/// </summary>
/// <param name="L"></param>
protected override void CreateEquationsAndSolvers(GridUpdateDataVaultBase L)
{
using (FuncTrace tr = new FuncTrace()) {
// create operator
// ===============
SpatialOperator LapaceIp;
{
double D = this.GridData.SpatialDimension;
double penalty_base = (T.Basis.Degree + 1) * (T.Basis.Degree + D) / D;
double penalty_factor = base.Control.penalty_poisson;
BoundaryCondMap <BoundaryType> PoissonBcMap = new BoundaryCondMap <BoundaryType>(this.GridData, this.Control.BoundaryValues, "T");
LapaceIp = new SpatialOperator(1, 1, QuadOrderFunc.SumOfMaxDegrees(), "T", "T");
var flux = new ipFlux(penalty_base * base.Control.penalty_poisson, this.GridData.Cells.cj, PoissonBcMap);
LapaceIp.EquationComponents["T"].Add(flux);
LapaceIp.Commit();
}
// Create Matrices
// ===============
{
// time measurement for matrix assembly
Stopwatch stw = new Stopwatch();
stw.Start();
// console
Console.WriteLine("creating sparse system for {0} DOF's ...", T.Mapping.Ntotal);
// quadrature domain
var volQrSch = new CellQuadratureScheme(true, CellMask.GetFullMask(this.GridData));
var edgQrSch = new EdgeQuadratureScheme(true, EdgeMask.GetFullMask(this.GridData));
#if DEBUG
// in DEBUG mode, we compare 'MsrMatrix' (old, reference implementation) and 'BlockMsrMatrix' (new standard)
var RefLaplaceMtx = new MsrMatrix(T.Mapping);
#endif
using (new BlockTrace("SipMatrixAssembly", tr)) {
LaplaceMtx = new BlockMsrMatrix(T.Mapping);
LaplaceAffine = new double[T.Mapping.LocalLength];
LapaceIp.ComputeMatrixEx(T.Mapping, null, T.Mapping,
LaplaceMtx, LaplaceAffine,
volQuadScheme: volQrSch, edgeQuadScheme: edgQrSch);
}
#if DEBUG
LaplaceAffine.ClearEntries();
LapaceIp.ComputeMatrixEx(T.Mapping, null, T.Mapping,
RefLaplaceMtx, LaplaceAffine,
volQuadScheme: volQrSch, edgeQuadScheme: edgQrSch);
MsrMatrix ErrMtx = RefLaplaceMtx.CloneAs();
ErrMtx.Acc(-1.0, LaplaceMtx);
double err = ErrMtx.InfNorm();
double infNrm = LaplaceMtx.InfNorm();
Console.WriteLine("Matrix comparison error: " + err + ", matrix norm is: " + infNrm);
Assert.Less(err, infNrm * 1e-10, "MsrMatrix2 comparison failed.");
#endif
stw.Stop();
Console.WriteLine("done {0} sec.", stw.Elapsed.TotalSeconds);
}
//double condNo = LaplaceMtx.condest(BatchmodeConnector.Flavor.Octave);
//Console.WriteLine("condition number: {0:0.####E-00} ",condNo);
}
}