/// <summary>
/// returns all external rows of <paramref name="M"/>
/// corresponding to ghost cells of <paramref name="map"/>,
/// which are located on other Mpi-ranks.
/// </summary>
/// <param name="map">Multigrid mapping</param>
/// <param name="M">matrix distributed according to <paramref name="map"/></param>
/// <returns></returns>
public static BlockMsrMatrix GetAllExternalRows(MultigridMapping map, BlockMsrMatrix M)
{
var extcells = map.AggGrid.iLogicalCells.NoOfExternalCells.ForLoop(i => i + map.LocalNoOfBlocks);
var SBS = new SubBlockSelector(map);
SBS.CellSelector(extcells, false);
var AllExtMask = new BlockMaskExt(SBS, 0);
var ExternalRows_BlockI0 = AllExtMask.GetAllSubMatrixCellOffsets();
var ExternalRows_BlockN = AllExtMask.GetAllSubMatrixCellLength();
var ExternalRowsIndices = AllExtMask.m_GlobalMask;
BlockPartitioning PermRow = new BlockPartitioning(ExternalRowsIndices.Count, ExternalRows_BlockI0, ExternalRows_BlockN, M.MPI_Comm, i0isLocal: true);
BlockMsrMatrix Perm = new BlockMsrMatrix(PermRow, M._RowPartitioning);
for (int iRow = 0; iRow < ExternalRowsIndices.Count; iRow++)
{
Debug.Assert(M._RowPartitioning.IsInLocalRange(ExternalRowsIndices[iRow]) == false);
Perm[iRow + PermRow.i0, ExternalRowsIndices[iRow]] = 1;
}
#if TEST
Perm.SaveToTextFileSparseDebug("Perm");
#endif
return(BlockMsrMatrix.Multiply(Perm, M));
}
/// <summary>
/// Get SubMatrix corresponding to this <see cref="BlockMask"/>.
/// With the ignore flags, coupling blocks can be left out (e.g. blocks containing level-set).
/// If <paramref name="ignoreCellCoupling"/> is set true, only diagonal blocks are concidered.
/// Probably slower than <see cref="GetSubBlockMatrix(BlockMsrMatrix)"/>.
/// </summary>
/// <param name="source">matrix to apply masking to</param>
/// <param name="ignoreCellCoupling">flag to ignore cell coupling</param>
/// <param name="ignoreVarCoupling">flag to ignore variable coupling</param>
/// <param name="ignoreSpecCoupling">flag to ignore species coupling</param>
/// <returns>sub block matrix</returns>
public BlockMsrMatrix GetSubBlockMatrix(BlockMsrMatrix source, bool ignoreCellCoupling, bool ignoreVarCoupling, bool ignoreSpecCoupling)
{
if (ignoreCellCoupling && m_includeExternalCells)
{
throw new NotImplementedException("Coupling of internal and external block is not concidered");
}
BlockMsrMatrix submatrix = null;
if (m_includeExternalCells)
{
BlockPartitioning extBlocking = new BlockPartitioning(BMLoc.LocalDOF + BMExt.LocalDOF, SubMatrixOffsets, SubMatrixLen, csMPI.Raw._COMM.SELF);
submatrix = new BlockMsrMatrix(extBlocking);
var ExtRowsTmp = m_ExtRows;
AuxGetSubBlockMatrix(submatrix, source, BMLoc, ignoreCellCoupling, ignoreVarCoupling, ignoreSpecCoupling);
AuxGetSubBlockMatrix(submatrix, ExtRowsTmp, BMExt, ignoreCellCoupling, ignoreVarCoupling, ignoreSpecCoupling);
}
else
{
int Loclength = BMLoc.LocalDOF;
var tmpN = BMLoc.GetAllSubMatrixCellLength();
var tmpi0 = BMLoc.GetAllSubMatrixCellOffsets();
BlockPartitioning localBlocking = new BlockPartitioning(Loclength, tmpi0.ToArray(), tmpN.ToArray(), csMPI.Raw._COMM.SELF, i0isLocal: true);
submatrix = new BlockMsrMatrix(localBlocking);
AuxGetSubBlockMatrix(submatrix, source, BMLoc, ignoreCellCoupling, ignoreVarCoupling, ignoreSpecCoupling);
}
Debug.Assert(submatrix != null);
return(submatrix);
}
/// <summary>
/// If you just want to get the <see cref="BlockMsrMatrix"/>, which corresponds to this <see cref="BlockMask"/>.
/// This is the method to choose! In addition, MPI communicator can be defined via <paramref name="comm"/>.
/// </summary>
/// <returns>submatrix on <paramref name="comm"/></returns>
public BlockMsrMatrix GetSubBlockMatrix(BlockMsrMatrix source, MPI_Comm comm)
{
if (source == null)
{
throw new ArgumentNullException();
}
if (source.NoOfRows < BMLoc.LocalDOF)
{
throw new ArgumentException();
}
BlockMsrMatrix target;
if (m_includeExternalCells)
{
BlockPartitioning targetBlocking = new BlockPartitioning(BMLoc.LocalDOF + BMExt.LocalDOF, SubMatrixOffsets, SubMatrixLen, comm);
//make an extended block dummy to fit local and external blocks
target = new BlockMsrMatrix(targetBlocking, targetBlocking);
//get the external rows via MPI exchange
var ExtRowsTmp = m_ExtRows;
int offset = BMLoc.m_GlobalMask.Count;
var extBlockRows = BMExt.m_GlobalMask.Count.ForLoop(i => i + offset);
var extBlockCols = BMExt.m_GlobalMask.Count.ForLoop(i => i + offset);
//ExtRowsTmp lives at the MPI-Communicator of the target, thus the global index is related to a new partitioning and has nothing to do with the partitioning of the multigrid operator ...
var GlobalIdxExtRows = BMExt.m_GlobalMask.Count.ForLoop(i => BMExt.m_LocalMask[i] - m_map.LocalLength);
for (int iGlob = 0; iGlob < GlobalIdxExtRows.Count(); iGlob++)
{
Debug.Assert(GlobalIdxExtRows[iGlob] < ExtRowsTmp._RowPartitioning.LocalLength);
GlobalIdxExtRows[iGlob] += ExtRowsTmp._RowPartitioning.i0;
Debug.Assert(ExtRowsTmp._RowPartitioning.IsInLocalRange(GlobalIdxExtRows[iGlob]));
}
//add local Block ...
source.WriteSubMatrixTo(target, BMLoc.m_GlobalMask, default(int[]), BMLoc.m_GlobalMask, default(int[]));
//add columns related to external rows ...
source.AccSubMatrixTo(1.0, target, BMLoc.m_GlobalMask, default(int[]), new int[0], default(int[]), BMExt.m_GlobalMask, extBlockCols);
//add external rows ...
ExtRowsTmp.AccSubMatrixTo(1.0, target, GlobalIdxExtRows, extBlockRows, BMLoc.m_GlobalMask, default(int[]), BMExt.m_GlobalMask, extBlockCols);
}
else
{
BlockPartitioning localBlocking = new BlockPartitioning(BMLoc.LocalDOF, SubMatrixOffsets, SubMatrixLen, csMPI.Raw._COMM.SELF, i0isLocal: true);
target = new BlockMsrMatrix(localBlocking);
source.AccSubMatrixTo(1.0, target, BMLoc.m_GlobalMask, default(int[]), BMLoc.m_GlobalMask, default(int[]));
}
Debug.Assert(target != null);
return(target);
}
public static BlockMsrMatrix GetCellCompMatrix(SelectionType SType, MultigridOperator mop, int iB)
{
int rank = mop.Mapping.MpiRank;
int iBlock = mop.Mapping.AggGrid.CellPartitioning.i0 + iB;
int i0 = mop.Mapping.GetBlockI0(iBlock);
//int jBlock = i0 + jB;
int R = mop.Mapping.GetBlockLen(iBlock);
//int C = mop.Mapping.GetBlockLen(jBlock);
bool ZwoSpecR = Math.Max(mop.Mapping.GetSubblkLen(0)[0], mop.Mapping.GetSubblkLen(1)[0]) == R;
//bool ZwoSpecC = (mop.Mapping.AggBasis[0].GetMaximalLength(DGdegree) + mop.Mapping.AggBasis[0].GetMaximalLength(DGdegree - 1)) == C;
SpeciesId A = ((XdgAggregationBasis)mop.Mapping.AggBasis[0]).UsedSpecies[0];
int Specpos = ((XdgAggregationBasis)mop.Mapping.AggBasis[0]).GetSpeciesIndex(iB, A);
int[] SubIdcR = GetSubIndices(SType, ZwoSpecR, Specpos);
//int[] SubIdcC = GetSubIndices(SType, ZwoSpecC);
for (int i = 0; i < SubIdcR.Length; i++)
{
Debug.Assert(SubIdcR[i] < R);
SubIdcR[i] += i0;
}
//for (int i = 0; i < SubIdcC.Length; i++) {
// Debug.Assert(SubIdcC[i] < C);
// SubIdcC[i] += i0;
//}
//return mop.OperatorMatrix.GetSubMatrix(SubIdcR, SubIdcC);
var part = new BlockPartitioning(SubIdcR.Length, new int[] { 0 }, new int[] { SubIdcR.Length }, csMPI.Raw._COMM.SELF);
BlockMsrMatrix sub = new BlockMsrMatrix(part);
mop.OperatorMatrix.WriteSubMatrixTo(sub, SubIdcR, default(int[]), SubIdcR, default(int[]));
return(sub);
//return mop.OperatorMatrix.GetSubMatrix(SubIdcR, SubIdcR);
}
public static BlockMsrMatrix ConvertToQuadraticBMsr(this BlockMsrMatrix M, int[] Colidx, bool isinternal)
{
Debug.Assert(M._RowPartitioning.LocalLength == Colidx.Length);
int NoOfBlocks = M._RowPartitioning.LocalNoOfBlocks;
int[] Offsets = new int[NoOfBlocks];
int[] Lengths = new int[NoOfBlocks];
int IdxOffset = M._RowPartitioning.i0;
int ColIdxOffset = M._ColPartitioning.i0;
for (int i = 0; i < NoOfBlocks; i++)
{
int iBlock = i + M._RowPartitioning.FirstBlock;
Offsets[i] = M._RowPartitioning.GetBlockI0(iBlock) - IdxOffset;
Lengths[i] = M._RowPartitioning.GetBlockLen(iBlock);
}
BlockPartitioning part = new BlockPartitioning(M._RowPartitioning.LocalLength, Offsets, Lengths, csMPI.Raw._COMM.SELF, true);
BlockMsrMatrix ret = new BlockMsrMatrix(part);
int[] RowISrc = M._RowPartitioning.LocalLength.ForLoop(i => i + IdxOffset);
//int[] ColISrc = M._ColPartitioning.LocalLength.ForLoop(i => Colidx[i]);
//if (ColISrc.Length < RowISrc.Length)
// ExtISrc = (RowISrc.Length - ColISrc.Length).ForLoop(i => Colidx[i+ ColISrc.Length]);
int[] ExtISrc = M._RowPartitioning.LocalLength.ForLoop(i => Colidx[i]);
int[] ExtITrg = M._RowPartitioning.LocalLength.ForLoop(i => i);
if (isinternal)
{
M.AccSubMatrixTo(1.0, ret, RowISrc, default(int[]), ExtISrc, default(int[]));
}
else
{
M.AccSubMatrixTo(1.0, ret, RowISrc, default(int[]), new int[0], default(int[]), ExtISrc, ExtITrg);
}
return(ret);
}
public void Init(MultigridOperator op)
{
using (new FuncTrace()) {
if (m_MgOp != null)
{
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// someone is trying to re-use this solver: see if the settings permit that
// ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if (op.LevelIndex != m_MgOp.LevelIndex)
{
throw new ArgumentException("Re-use on different level not possible.");
}
if (!this.MtxFull._RowPartitioning.EqualsPartition(op.OperatorMatrix._RowPartitioning))
{
throw new ArgumentException("Matrix has changed, unable to re-use");
}
if (!this.MtxFull._ColPartitioning.EqualsPartition(op.OperatorMatrix._ColPartitioning))
{
throw new ArgumentException("Matrix has changed, unable to re-use");
}
#if DEBUG
if (!object.ReferenceEquals(this.MtxFull, op.OperatorMatrix))
{
BlockMsrMatrix Check = this.MtxFull.CloneAs();
Check.Acc(-1.0, op.OperatorMatrix);
if (Check.InfNorm() != 0.0)
{
throw new ArgumentException("Matrix has changed, unable to re-use");
}
}
#endif
if (this.m_BlockingStrategy.GetNoOfBlocks(op) != this.blockSolvers.Count())
{
throw new ArgumentException("Blocking, unable to re-use");
}
return;
}
var Mop = op.OperatorMatrix;
var MgMap = op.Mapping;
this.m_MgOp = op;
int myMpiRank = MgMap.MpiRank;
int myMpisize = MgMap.MpiSize;
if (!Mop.RowPartitioning.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Row partitioning mismatch.");
}
if (!Mop.ColPartition.EqualsPartition(MgMap.Partitioning))
{
throw new ArgumentException("Column partitioning mismatch.");
}
var ag = MgMap.AggGrid;
int JComp = ag.iLogicalCells.NoOfLocalUpdatedCells;
int JGhost = ag.iLogicalCells.NoOfExternalCells;
#if DEBUG
ilPSP.Connectors.Matlab.BatchmodeConnector matlab;
if (m_MatlabParalellizationCheck)
{
matlab = new ilPSP.Connectors.Matlab.BatchmodeConnector();
}
else
{
matlab = null;
}
#endif
//Mop.Clear();
//for(int i = Mop.RowPartitioning.i0; i < Mop.RowPartitioning.iE; i++) {
// Mop[i, i] = i + 1;
//}
// get cell blocks
// ===============
var _Blocks = this.m_BlockingStrategy.GetBlocking(op);
int NoOfSchwzBlocks = _Blocks.Count();
// test cell blocks
// ================
#if DEBUG
{
// ensure that each cell is used exactly once, among all blocks
bool[] test = new bool[ag.iLogicalCells.NoOfLocalUpdatedCells];
foreach (var bi in _Blocks)
{
foreach (int j in bi)
{
Debug.Assert(test[j] == false);
test[j] = true;
}
;
}
for (int i = 0; i < test.Length; i++)
{
Debug.Assert(test[i] == true);
}
}
#endif
// extend blocks according to desired overlap
// ==========================================
{
BitArray marker = new BitArray(JComp + JGhost);
if (Overlap < 0)
{
throw new ArgumentException();
}
if (Overlap > 0)
{
if (Overlap > 1 && Mop.RowPartitioning.MpiSize > 1)
{
throw new NotSupportedException("In MPI parallel runs, the maximum supported overlap for the Schwarz preconditioner is 1.");
}
foreach (List <int> bi in _Blocks) // loop over blocks...
{
marker.SetAll(false); // marks all cells which are members of the block
foreach (int jcomp in bi)
{
marker[jcomp] = true;
}
// determine overlap regions
for (int k = 0; k < Overlap; k++)
{
int Jblock = bi.Count;
for (int j = 0; j < Jblock; j++)
{
int jCell = bi[j];
int[] Neighs = ag.iLogicalCells.CellNeighbours[jCell];
foreach (int jNeigh in Neighs)
{
if (marker[jNeigh] == false)
{
// neighbor cell is not already a member of the block
// => add it.
bi.Add(jNeigh);
marker[jNeigh] = true;
}
}
}
}
bi.Sort();
}
}
BlockCells = _Blocks.Select(list => list.ToArray()).ToArray();
}
// convert cell blocks to DOF blocks
// =================================
List <int>[] BlkIdx_gI_lR; // for each Schwarz block, (global) indices in the local range
List <int>[] BlkIdx_gI_eR; // for each Schwarz block, (global) indices of external rows and columns
List <int>[] TempRowIdx_gI; // for each Schwarz block, (global) indices into the temporary matrix
List <int>[] BlkIdx_lI_eR; // for each Schwarz block, (local) indices of external rows and columns
List <int>[] LocalBlocks_i0, LocalBlocks_N; // blocking of the Schwarz-Blocks.
// for matrix 'ExternalRowsTemp': which rows of 'Mop' are required locally
List <int> ExternalRowsIndices, ExternalRows_BlockI0, ExternalRows_BlockN;
{
int Jup = MgMap.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
int Jgh = MgMap.AggGrid.iLogicalCells.NoOfExternalCells;
int LocalizedBlockCounter = 0;
BlkIdx_gI_lR = NoOfSchwzBlocks.ForLoop(iPart => new List <int>(BlockCells[iPart].Length * MgMap.MaximalLength));
BlkIdx_gI_eR = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
LocalBlocks_i0 = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
LocalBlocks_N = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
TempRowIdx_gI = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
BlkIdx_lI_eR = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
ExternalRowsIndices = new List <int>();
ExternalRows_BlockI0 = new List <int>();
ExternalRows_BlockN = new List <int>();
for (int iPart = 0; iPart < NoOfSchwzBlocks; iPart++) // loop over parts...
{
int[] bc = BlockCells[iPart];
var biI = BlkIdx_gI_lR[iPart];
var biE = BlkIdx_gI_eR[iPart];
var l1 = TempRowIdx_gI[iPart];
var l2 = BlkIdx_lI_eR[iPart];
var LBBi0 = LocalBlocks_i0[iPart];
var LBBN = LocalBlocks_N[iPart];
int Jblock = bc.Length;
int anotherCounter = 0;
for (int jblk = 0; jblk < Jblock; jblk++) // loop over cells in blocks...
{
int j = bc[jblk];
int N = MgMap.GetLength(j);
if (j < Jup)
{
// locally updated cell
int i0 = MgMap.GlobalUniqueIndex(0, j, 0);
for (int n = 0; n < N; n++)
{
biI.Add(i0 + n);
}
}
else
{
// external cell
int i0E = MgMap.GlobalUniqueIndex(0, j, 0); //
int i0L = MgMap.LocalUniqueIndex(0, j, 0); //
ExternalRows_BlockI0.Add(LocalizedBlockCounter);
ExternalRows_BlockN.Add(N);
//LEBi0.Add(LocalizedBlockCounter);
//LEBn.Add(N);
for (int n = 0; n < N; n++)
{
biE.Add(i0E + n);
ExternalRowsIndices.Add(i0E + n);
l1.Add(LocalizedBlockCounter + n);
l2.Add(i0L + n);
Debug.Assert(Mop._RowPartitioning.FindProcess(i0E + n) != myMpiRank);
}
LocalizedBlockCounter += N;
}
LBBi0.Add(anotherCounter);
LBBN.Add(N);
anotherCounter += N;
}
}
//this.BlockIndices = _LocallyStoredBlockIndices.Select(bi => bi.ToArray()).ToArray();
}
// get rows for blocks that use external cells
// ===========================================
#if DEBUG
{
if (Overlap == 0)
{
Debug.Assert(ExternalRowsIndices.Count == 0);
Debug.Assert(ExternalRows_BlockI0.Count == 0);
Debug.Assert(ExternalRows_BlockN.Count == 0);
}
foreach (var bi in BlkIdx_gI_lR)
{
foreach (int idx in bi)
{
Debug.Assert(idx >= m_MgOp.Mapping.i0);
Debug.Assert(idx < m_MgOp.Mapping.iE);
}
}
foreach (var ei in BlkIdx_gI_eR)
{
foreach (int idx in ei)
{
Debug.Assert(idx < m_MgOp.Mapping.i0 || idx >= m_MgOp.Mapping.iE);
}
}
int LL = m_MgOp.Mapping.LocalLength;
int jMax = m_MgOp.Mapping.AggGrid.iLogicalCells.NoOfCells - 1;
int LE = m_MgOp.Mapping.LocalUniqueIndex(0, jMax, 0) + m_MgOp.Mapping.GetLength(jMax);
foreach (var ci in BlkIdx_lI_eR)
{
foreach (int idx in ci)
{
Debug.Assert(idx >= LL);
Debug.Assert(idx < LE);
}
}
if (m_MatlabParalellizationCheck)
{
int globalBlockCounter = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
if (rankCounter == myMpiRank)
{
Debug.Assert(rank_NoBlks == NoOfSchwzBlocks);
}
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
double[] vec;
if (rankCounter == myMpiRank)
{
vec = ArrayTools.Cat(BlkIdx_gI_lR[iBlock], BlkIdx_gI_eR[iBlock]).Select(ii => ((double)(ii + 1))).ToArray();
}
else
{
vec = new double[0];
}
matlab.PutVector(vec, string.Format("BlockIdx{0}", globalBlockCounter));
globalBlockCounter++;
csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD);
}
}
}
}
#endif
BlockMsrMatrix ExternalRowsTemp;
if (myMpisize > 1 && Overlap > 0)
{
//int NoOfLocalRows = _ExternalBlockIndices.Sum(L => L.Count);
BlockPartitioning PermRow = new BlockPartitioning(ExternalRowsIndices.Count, ExternalRows_BlockI0, ExternalRows_BlockN, Mop.MPI_Comm, i0isLocal: true);
// Remark: we use a permutation matrix for MPI-exchange of rows
BlockMsrMatrix Perm = new BlockMsrMatrix(PermRow, Mop._RowPartitioning);
for (int iRow = 0; iRow < ExternalRowsIndices.Count; iRow++)
{
Debug.Assert(Mop._RowPartitioning.IsInLocalRange(ExternalRowsIndices[iRow]) == false);
Perm[iRow + PermRow.i0, ExternalRowsIndices[iRow]] = 1;
}
ExternalRowsTemp = BlockMsrMatrix.Multiply(Perm, Mop);
#if DEBUG
if (m_MatlabParalellizationCheck)
{
matlab.PutSparseMatrix(Perm, "Perm");
matlab.PutSparseMatrix(ExternalRowsTemp, "ExternalRowsTemp");
}
#endif
}
else
{
ExternalRowsTemp = null;
}
ExternalRowsIndices = null;
ExternalRows_BlockI0 = null;
ExternalRows_BlockN = null;
// create solvers
// ==============
{
blockSolvers = new ISparseSolver[NoOfSchwzBlocks];
#if DEBUG
List <BlockMsrMatrix> Blocks = new List <BlockMsrMatrix>();
#endif
for (int iPart = 0; iPart < NoOfSchwzBlocks; iPart++)
{
var bi = BlkIdx_gI_lR[iPart];
int Bsz;
if (MgMap.MinimalLength == MgMap.MaximalLength)
{
Bsz = MgMap.MaximalLength;
}
else
{
Bsz = 1;
}
var l1 = TempRowIdx_gI[iPart];
//if (M.RowPartitioning.MpiSize > 1) {
// int i0Proc = M.RowPartitioning.i0;
// bi = bi.CloneAs();
// for (int i = 0; i < bi.Length; i++) {
// bi[i] += i0Proc;
// }
//}
BlockPartitioning localBlocking = new BlockPartitioning(bi.Count + l1.Count, LocalBlocks_i0[iPart], LocalBlocks_N[iPart], csMPI.Raw._COMM.SELF);
if (l1.Count > 0)
{
// convert the indices into 'ExternalRowsTemp' to global indices
int l1L = l1.Count;
int offset = ExternalRowsTemp._RowPartitioning.i0;
for (int i = 0; i < l1L; i++)
{
l1[i] += offset;
}
}
BlockMsrMatrix Block = new BlockMsrMatrix(localBlocking, localBlocking);// bi.Length, bi.Length, Bsz, Bsz);
Mop.WriteSubMatrixTo(Block, bi, default(int[]), bi, default(int[]));
if (l1.Count > 0)
{
int offset = bi.Count;
int[] targRows = l1.Count.ForLoop(i => i + offset);
var biE = BlkIdx_gI_eR[iPart];
int[] extTargCols = biE.Count.ForLoop(i => i + offset);
Mop.AccSubMatrixTo(1.0, Block, bi, default(int[]), new int[0], default(int[]), biE, extTargCols);
ExternalRowsTemp.AccSubMatrixTo(1.0, Block, l1, targRows, bi, default(int[]), biE, extTargCols);
}
#if DEBUG
if (m_MatlabParalellizationCheck)
{
Blocks.Add(Block);
}
#endif
blockSolvers[iPart] = new PARDISOSolver()
{
CacheFactorization = true,
UseDoublePrecision = false
};
//blockSolvers[iPart] = new FullDirectSolver();
//blockSolvers[iPart] = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
blockSolvers[iPart].DefineMatrix(Block);
}
#if DEBUG
if (m_MatlabParalellizationCheck)
{
int globalBlockCounter = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
BlockMsrMatrix Block;
if (rankCounter == myMpiRank)
{
Block = Blocks[iBlock];
}
else
{
Block = null;
}
matlab.PutSparseMatrix(Block, string.Format("Block{0}", globalBlockCounter));
globalBlockCounter++;
csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD);
}
}
}
#endif
}
// Record required indices
// =======================
{
this.BlockIndices_Local = new int[NoOfSchwzBlocks][];
this.BlockIndices_External = new int[NoOfSchwzBlocks][];
int LocalI0 = MgMap.i0;
int LocalLength = MgMap.LocalLength;
for (int iBlock = 0; iBlock < NoOfSchwzBlocks; iBlock++)
{
var _bi = BlkIdx_gI_lR[iBlock];
int L = _bi.Count;
int[] bil = new int[L];
this.BlockIndices_Local[iBlock] = bil;
for (int l = 0; l < L; l++)
{
bil[l] = _bi[l] - LocalI0;
Debug.Assert(bil[l] >= 0);
Debug.Assert(bil[l] < MgMap.LocalLength);
}
var _biE = BlkIdx_lI_eR[iBlock];
if (_biE.Count > 0)
{
this.BlockIndices_External[iBlock] = _biE.ToArray();
}
}
}
this.MtxFull = Mop;
if (CoarseSolver != null)
{
CoarseSolver.Init(op.CoarserLevel);
}
// Debug & Test-Code
// =================
#if DEBUG
if (m_MatlabParalellizationCheck)
{
Console.WriteLine("Matlab dir: " + matlab.WorkingDirectory);
matlab.PutSparseMatrix(Mop, "Full");
int GlobalNoOfBlocks = NoOfSchwzBlocks.MPISum();
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
matlab.Cmd("BlockErr({0} + 1, 1) = norm( Block{0} - Full( BlockIdx{0}, BlockIdx{0} ), inf );", iGlbBlock);
}
Random rnd = new Random(myMpiRank);
double[] testRHS = new double[MgMap.LocalLength];
for (int i = 0; i < testRHS.Length; i++)
{
testRHS[i] = rnd.NextDouble();
}
matlab.PutVector(testRHS, "testRHS");
MPIexchange <double[]> ResExchange = new MPIexchange <double[]>(MgMap, testRHS);
ResExchange.TransceiveStartImReturn();
ResExchange.TransceiveFinish(0.0);
int offset = MgMap.LocalLength;
int g = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
double[] SubVec;
if (rankCounter == myMpiRank)
{
int LL = this.BlockIndices_Local[iBlock].Length;
int LE;
if (this.BlockIndices_External[iBlock] != null)
{
LE = this.BlockIndices_External[iBlock].Length;
}
else
{
LE = 0;
}
int L = LL + LE;
SubVec = new double[L];
for (int i = 0; i < LL; i++)
{
SubVec[i] = testRHS[this.BlockIndices_Local[iBlock][i]];
}
if (LE > 0)
{
for (int i = 0; i < LE; i++)
{
SubVec[i + LL] = ResExchange.Vector_Ext[this.BlockIndices_External[iBlock][i] - offset];
}
}
}
else
{
SubVec = new double[0];
}
matlab.PutVector(SubVec, "SubVec" + g);
g++;
}
}
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
matlab.Cmd("RhsErr({0} + 1, 1) = norm( SubVec{0} - testRHS( BlockIdx{0} ), inf );", iGlbBlock);
}
double[] testX = new double[testRHS.Length];
MPIexchangeInverse <double[]> XExchange = new MPIexchangeInverse <double[]>(MgMap, testX);
g = 0;
for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
{
int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
{
if (rankCounter == myMpiRank)
{
int LL = this.BlockIndices_Local[iBlock].Length;
int LE;
if (this.BlockIndices_External[iBlock] != null)
{
LE = this.BlockIndices_External[iBlock].Length;
}
else
{
LE = 0;
}
int L = LL + LE;
for (int i = 0; i < LL; i++)
{
testX[this.BlockIndices_Local[iBlock][i]] += (g + 1);
}
if (LE > 0)
{
for (int i = 0; i < LE; i++)
{
XExchange.Vector_Ext[this.BlockIndices_External[iBlock][i] - offset] += (g + 1);
}
}
}
else
{
//nop
}
g++;
}
}
XExchange.TransceiveStartImReturn();
XExchange.TransceiveFinish(1.0);
matlab.Cmd("testXref = zeros({0},1);", MgMap.TotalLength);
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
matlab.Cmd("testXref(BlockIdx{0},1) = testXref(BlockIdx{0},1) + ({0} + 1);", iGlbBlock);
}
matlab.PutVector(testX, "testX");
matlab.Cmd("testXErr = norm(testX - testXref, inf);");
MultidimensionalArray BlockErr = MultidimensionalArray.Create(GlobalNoOfBlocks, 1);
MultidimensionalArray RhsErr = MultidimensionalArray.Create(GlobalNoOfBlocks, 1);
MultidimensionalArray testXErr = MultidimensionalArray.Create(1, 1);
matlab.GetMatrix(BlockErr, "BlockErr");
matlab.GetMatrix(RhsErr, "RhsErr");
matlab.GetMatrix(testXErr, "testXErr");
matlab.Execute();
for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
{
Console.WriteLine("Block #{0} Error (external? ) " + BlockErr[iGlbBlock, 0], iGlbBlock);
Console.WriteLine("RHS #{0} Error " + RhsErr[iGlbBlock, 0], iGlbBlock);
Debug.Assert(BlockErr[iGlbBlock, 0] == 0);
Debug.Assert(RhsErr[iGlbBlock, 0] == 0);
}
Console.WriteLine("X Error " + testXErr[0, 0]);
Debug.Assert(testXErr[0, 0] == 0.0);
matlab.Dispose();
}
#endif
}
}
/// <summary>
/// ~
/// </summary>
public void Init(MultigridOperator op)
{
//System.Threading.Thread.Sleep(10000);
//ilPSP.Environment.StdoutOnlyOnRank0 = false;
m_op = op;
var Map = op.Mapping;
int NoVars = Map.AggBasis.Length;
int j0 = Map.FirstBlock;
int J = Map.LocalNoOfBlocks;
int[] degs = m_op.Degrees;
for (int ideg = 0; ideg < degs.Length; ideg++)
{
if (degs[ideg] == 0)
{
throw new ArgumentException(String.Format("DGdegree for Variable {0} is 0, p-multigrid not possible", ideg));
}
}
var BS = Map.AggBasis;
if (UseHiOrderSmoothing)
{
HighOrderBlocks_LU = new MultidimensionalArray[J];
HighOrderBlocks_LUpivots = new int[J][];
HighOrderBlocks_indices = new int[J][];
}
var GsubIdx = new List <int>();
var LsubIdx = new List <int>();
var lowLocalBlocks_i0 = new List <int>();
var lowLocalBlocks__N = new List <int>();
int cnt = 0;
/*
* var debugerSW = new StreamWriter(String.Concat("debug_of_", ilPSP.Environment.MPIEnv.MPI_Rank));
* debugerSW.WriteLine("proc {0} reporting ...",ilPSP.Environment.MPIEnv.MPI_Rank);
* debugerSW.WriteLine("Num of Blocks {0}",HighOrderBlocks_LUpivots.Length);
*/
for (int jLoc = 0; jLoc < J; jLoc++)
{
lowLocalBlocks_i0.Add(cnt);
lowLocalBlocks__N.Add(0);
var LhiIdx = new List <int>();
var IdxHighBlockOffset = new int[NoVars][];
var IdxHighOffset = new int[NoVars][];
int NpHiTot = 0;
for (int iVar = 0; iVar < NoVars; iVar++)
{
int pReq;
if (degs[iVar] <= 1)
{
pReq = 0;
}
else
{
pReq = CoarseLowOrder;
}
int Np1 = BS[iVar].GetLength(jLoc, pReq);
int Np = BS[iVar].GetLength(jLoc, degs[iVar]);
lowLocalBlocks__N[jLoc] += Np1;
int NoOfSpc = BS[iVar].GetNoOfSpecies(jLoc);
int NpBase = Np / NoOfSpc;
int NpBaseLow = Np1 / NoOfSpc;
IdxHighBlockOffset[iVar] = new int[NoOfSpc + 1];
IdxHighOffset[iVar] = new int[NoOfSpc];
for (int iSpc = 0; iSpc < NoOfSpc; iSpc++)
{
int n = 0;
int cellOffset = NpBase * iSpc;
IdxHighOffset[iVar][iSpc] = Map.GlobalUniqueIndex(iVar, jLoc, cellOffset + NpBaseLow);
for (; n < NpBaseLow; n++)
{
int Lidx = Map.LocalUniqueIndex(iVar, jLoc, n + cellOffset);
LsubIdx.Add(Lidx); //local block mapping Coarse Matrix (low order entries) to original matrix
int Gidx = Map.GlobalUniqueIndex(iVar, jLoc, n + cellOffset);
GsubIdx.Add(Gidx); //global mapping Coarse Matrix (low order entries) to original matrix
}
for (; n < NpBase; n++)
{
int Lidx = Map.LocalUniqueIndex(iVar, jLoc, n + cellOffset);
LhiIdx.Add(Lidx); //local block mapping of high order entries to original matrix
//int Gidx = Map.GlobalUniqueIndex(iVar, jLoc, n);
//GhiIdx.Add(Gidx);
}
IdxHighBlockOffset[iVar][iSpc] = NpHiTot;
NpHiTot += (NpBase - NpBaseLow);
}
IdxHighBlockOffset[iVar][NoOfSpc] = NpHiTot;
Debug.Assert(GsubIdx.Last() == Map.GlobalUniqueIndex(iVar, jLoc, (NoOfSpc - 1) * NpBase + NpBaseLow - 1));
Debug.Assert(LhiIdx.Last() == Map.LocalUniqueIndex(iVar, jLoc, NoOfSpc * NpBase - 1));
}
// Save high order blocks for later smoothing
if (NpHiTot > 0)
{
HighOrderBlocks_LU[jLoc] = MultidimensionalArray.Create(NpHiTot, NpHiTot);
HighOrderBlocks_indices[jLoc] = LhiIdx.ToArray();
for (int iVar = 0; iVar < NoVars; iVar++)
{
for (int jVar = 0; jVar < NoVars; jVar++)
{
for (int iSpc = 0; iSpc < BS[jVar].GetNoOfSpecies(jLoc); iSpc++)
{
int i0_hi = IdxHighOffset[iVar][iSpc];
int j0_hi = IdxHighOffset[jVar][iSpc];
int Row_i0 = IdxHighBlockOffset[iVar][iSpc];
int Col_i0 = IdxHighBlockOffset[jVar][iSpc];
int Row_ie = IdxHighBlockOffset[iVar][iSpc + 1];
int col_ie = IdxHighBlockOffset[jVar][iSpc + 1];
//debugerSW.WriteLine("Block {0}: i0={1} j0={2} iVar={3}", jLoc, i0_hi, j0_hi, iVar);
m_op.OperatorMatrix.ReadBlock(i0_hi, j0_hi,
HighOrderBlocks_LU[jLoc].ExtractSubArrayShallow(new int[] { Row_i0, Col_i0 }, new int[] { Row_ie - 1, col_ie - 1 }));
}
}
}
HighOrderBlocks_LUpivots[jLoc] = new int[NpHiTot];
HighOrderBlocks_LU[jLoc].FactorizeLU(HighOrderBlocks_LUpivots[jLoc]);
}
cnt += lowLocalBlocks__N[jLoc];
}
m_LsubIdx = LsubIdx.ToArray();
BlockPartitioning localBlocking = new BlockPartitioning(GsubIdx.Count, lowLocalBlocks_i0, lowLocalBlocks__N, Map.MPI_Comm, i0isLocal: true);
var P01SubMatrix = new BlockMsrMatrix(localBlocking);
op.OperatorMatrix.AccSubMatrixTo(1.0, P01SubMatrix, GsubIdx, default(int[]), GsubIdx, default(int[]));
intSolver = new PARDISOSolver()
{
CacheFactorization = true,
UseDoublePrecision = false
};
intSolver.DefineMatrix(P01SubMatrix);
/*
* LsubIdx.SaveToTextFileDebug("LsubIdx");
* GsubIdx.SaveToTextFileDebug("GsubIdx");
* P01SubMatrix.SaveToTextFileSparseDebug("lowM");
* m_op.OperatorMatrix.SaveToTextFileSparseDebug("M");
* P01SubMatrix.SaveToTextFileSparse("lowM_full");
* m_op.OperatorMatrix.SaveToTextFileSparse("M_full");
*
* debugerSW.WriteLine("Dim of lowMatrix: {0}",GsubIdx.Count);
* debugerSW.Flush();
* debugerSW.Close();
*/
}
