/// <summary>
/// returns all external rows of <paramref name="M"/>
/// corresponding to ghost cells of <paramref name="map"/>,
/// which are located on other Mpi-ranks.
/// </summary>
/// <param name="map">Multigrid mapping</param>
/// <param name="M">matrix distributed according to <paramref name="map"/></param>
/// <returns></returns>
public static BlockMsrMatrix GetAllExternalRows(MultigridMapping map, BlockMsrMatrix M)
{
var extcells = map.AggGrid.iLogicalCells.NoOfExternalCells.ForLoop(i => i + map.LocalNoOfBlocks);
var SBS = new SubBlockSelector(map);
SBS.CellSelector(extcells, false);
var AllExtMask = new BlockMaskExt(SBS, 0);
var ExternalRows_BlockI0 = AllExtMask.GetAllSubMatrixCellOffsets();
var ExternalRows_BlockN = AllExtMask.GetAllSubMatrixCellLength();
var ExternalRowsIndices = AllExtMask.m_GlobalMask;
BlockPartitioning PermRow = new BlockPartitioning(ExternalRowsIndices.Count, ExternalRows_BlockI0, ExternalRows_BlockN, M.MPI_Comm, i0isLocal: true);
BlockMsrMatrix Perm = new BlockMsrMatrix(PermRow, M._RowPartitioning);
for (int iRow = 0; iRow < ExternalRowsIndices.Count; iRow++)
{
Debug.Assert(M._RowPartitioning.IsInLocalRange(ExternalRowsIndices[iRow]) == false);
Perm[iRow + PermRow.i0, ExternalRowsIndices[iRow]] = 1;
}
#if TEST
Perm.SaveToTextFileSparseDebug("Perm");
#endif
return(BlockMsrMatrix.Multiply(Perm, M));
}
public BlockMaskLoc(SubBlockSelector sbs) : base(sbs, csMPI.Raw._COMM.SELF)
{
base.GenerateAllMasks();
foreach (var idx in this.m_GlobalMask)
{
Debug.Assert(idx >= m_map.i0);
Debug.Assert(idx < m_map.iE);
}
}
private void AssignXdgBlocksModification(SubBlockSelector sbs, MultigridOperator op, bool IsLowSelector)
{
var Filter = sbs.ModeFilter;
Func <int, int, int, int, bool> Modification = delegate(int iCell, int iVar, int iSpec, int pDeg) {
int NoOfSpec = op.Mapping.AggBasis[0].GetNoOfSpecies(iCell);
if (NoOfSpec >= 2)
{
return(IsLowSelector);
}
else
{
return(Filter(iCell, iVar, iSpec, pDeg));
}
};
sbs.ModeSelector(Modification);
}
/// <summary>
/// Generates Block Mask (index lists) from Sub block selection based on a multigrid mapping.
/// abstract parts are individuallized by child classes: <see cref="BlockMask.BlockMaskLoc"/> and <see cref="BlockMask.BlockMaskExt"/>
/// </summary>
/// <param name="SBS"></param>
public BlockMaskBase(SubBlockSelector SBS, MPI_Comm MPIcomm)
{
m_map = SBS.GetMapping;
m_sbs = SBS;
m_AggBS = m_map.AggBasis;
m_DGdegree = m_map.DgDegree;
m_Ni0 = Ni0Gen();
m_NoOfVariables = m_AggBS.Length;
//Testen ob es cells gibt wo Var<>NoOfVar, das würde dementsprechend auch m_DG beeinflussen
m_NoOfSpecies = new int[m_NoOfCells][];
for (int iCell = 0; iCell < m_NoOfCells; iCell++)
{
m_NoOfSpecies[iCell] = new int[m_NoOfVariables];
for (int iVar = 0; iVar < m_NoOfVariables; iVar++)
{
m_NoOfSpecies[iCell][iVar] = m_AggBS[iVar].GetNoOfSpecies(iCell + m_CellOffset);
}
}
}
/// <summary>
/// generates a masking of subblocks within a <see cref="MultigridMapping"/> according to <see cref="SubBlockSelector"/> <paramref name="sbs"/>
/// enables applying mask onto matrices and vectors, which comply with this <see cref="MultigridMapping"/>.
/// (e.g. sub matrix generation, sub vector extraction, etc.).
/// The smallest unit are dg blocks.
/// The masking operates on local blocks available on this proc per default,
/// which can be exceeded to external blocks by providing external rows: <paramref name="ExtRows"/>.
/// Ghost cells (covert by <see cref="BoSSS.Foundation.Grid.ILogicalCellData.NoOfExternalCells"/>) can be acquired by <see cref="GetAllExternalRows"/>.
/// </summary>
/// <param name="sbs">sub block selection defined by dev</param>
/// <param name="ExtRows">external rows collected from other MPI-processes on this proc</param>
public BlockMask(SubBlockSelector sbs, BlockMsrMatrix ExtRows = null)
{
m_map = sbs.GetMapping;
m_ExtRows = ExtRows;
m_includeExternalCells = (ExtRows != null) && m_map.MpiSize > 1;
BMLoc = new BlockMaskLoc(sbs);
if (m_includeExternalCells)
{
BMExt = new BlockMaskExt(sbs, BMLoc.LocalDOF);
SetThisShitUp(new BlockMaskBase[] { BMLoc, BMExt });
}
else
{
SetThisShitUp(new BlockMaskBase[] { BMLoc });
}
#if Debug
CheckIndices();
#endif
}
public BlockMaskExt(SubBlockSelector SBS, int LocMaskOffset) : base(SBS, SBS.GetMapping.MPI_Comm)
{
// must be set before mask generation
m_LocOffset = LocMaskOffset;
m_extLocLen = m_map.GetLocalLength_Ext();
base.GenerateAllMasks();
foreach (int idx in this.m_GlobalMask)
{
Debug.Assert(idx < m_map.i0 || idx >= m_map.iE);
}
int LL = m_map.LocalLength;
int jMax = m_map.AggGrid.iLogicalCells.Count - 1;
int LE = m_map.LocalUniqueIndex(0, jMax, 0) + m_map.GetLength(jMax);
foreach (int idx in this.m_LocalMask)
{
Debug.Assert(idx >= LL);
Debug.Assert(idx < LE);
}
}
private void ModifyHighSelector(SubBlockSelector sbs, MultigridOperator op)
{
AssignXdgBlocksModification(sbs, op, false);
}
private void ModifyLowSelector(SubBlockSelector sbs, MultigridOperator op)
{
AssignXdgBlocksModification(sbs, op, true);
}
/// <summary>
/// Krankplätze müssen verdichtet werden
/// -Kranführer Ronny, ProSieben Reportage
/// </summary>
public void Init(MultigridOperator op)
{
// //System.Threading.Thread.Sleep(10000);
// //ilPSP.Environment.StdoutOnlyOnRank0 = false;
m_op = op;
if (m_CoarseLowOrder > m_op.Mapping.DgDegree.Max())
{
throw new ArgumentOutOfRangeException("CoarseLowOrder is higher than maximal DG degree");
}
#if TEST
var debugerSW = new StreamWriter(String.Concat("debug_of_", ilPSP.Environment.MPIEnv.MPI_Rank));
Console.WriteLine("variable TEST is defined");
//debugerSW.WriteLine("proc {0} reporting Num of Blocks {1}", ilPSP.Environment.MPIEnv.MPI_Rank, HighOrderBlocks_LUpivots.Length);
#endif
int D = this.m_op.GridData.SpatialDimension;
var DGlowSelect = new SubBlockSelector(op.Mapping);
Func <int, int, int, int, bool> lowFilter = (int iCell, int iVar, int iSpec, int pDeg) => pDeg <= (iVar != D ? CoarseLowOrder : CoarseLowOrder - 1);
DGlowSelect.ModeSelector(lowFilter);
if (AssignXdGCellsToLowBlocks)
{
ModifyLowSelector(DGlowSelect, op);
}
lMask = new BlockMask(DGlowSelect);
m_lowMaskLen = lMask.GetNoOfMaskedRows;
if (UseHiOrderSmoothing)
{
var DGhighSelect = new SubBlockSelector(op.Mapping);
Func <int, int, int, int, bool> highFilter = (int iCell, int iVar, int iSpec, int pDeg) => pDeg > (iVar != D ? CoarseLowOrder : CoarseLowOrder - 1);
DGhighSelect.ModeSelector(highFilter);
if (AssignXdGCellsToLowBlocks)
{
ModifyHighSelector(DGhighSelect, op);
}
hMask = new BlockMask(DGhighSelect);
m_highMaskLen = hMask.GetNoOfMaskedRows;
BlockMsrMatrix P01HiMatrix = null;
if (UseDiagonalPmg)
{
HighOrderBlocks_LU = hMask.GetDiagonalBlocks(op.OperatorMatrix, false, false);
int NoOfBlocks = HighOrderBlocks_LU.Length;
HighOrderBlocks_LUpivots = new int[NoOfBlocks][];
for (int jLoc = 0; jLoc < NoOfBlocks; jLoc++)
{
int len = HighOrderBlocks_LU[jLoc].NoOfRows;
HighOrderBlocks_LUpivots[jLoc] = new int[len];
HighOrderBlocks_LU[jLoc].FactorizeLU(HighOrderBlocks_LUpivots[jLoc]);
}
}
else
{
P01HiMatrix = hMask.GetSubBlockMatrix(op.OperatorMatrix, csMPI.Raw._COMM.SELF);
hiSolver = new PARDISOSolver()
{
CacheFactorization = true,
UseDoublePrecision = true, // keep it true, experiments showed, that this leads to fewer iterations
SolverVersion = Parallelism.OMP
};
hiSolver.DefineMatrix(P01HiMatrix);
}
}
var P01SubMatrix = lMask.GetSubBlockMatrix(op.OperatorMatrix, csMPI.Raw._COMM.WORLD);
intSolver = new PARDISOSolver()
{
CacheFactorization = true,
UseDoublePrecision = false, // no difference towards =true observed for XDGPoisson
SolverVersion = Parallelism.OMP
};
intSolver.DefineMatrix(P01SubMatrix);
#if TEST
P01SubMatrix.SaveToTextFileSparseDebug("lowM");
P01SubMatrix.SaveToTextFileSparse("lowM_full");
if (!UseDiagonalPmg)
{
//P01HiMatrix.SaveToTextFileSparseDebug("hiM");
//P01HiMatrix.SaveToTextFileSparse("hiM_full");
}
m_op.OperatorMatrix.SaveToTextFileSparseDebug("M");
m_op.OperatorMatrix.SaveToTextFileSparse("M_full");
debugerSW.Flush();
debugerSW.Close();
long[] bla = m_op.BaseGridProblemMapping.GridDat.CurrentGlobalIdPermutation.Values;
bla.SaveToTextFileDebug("permutation_");
List <int> BlockI0 = new List <int>();
List <int> Block_N = new List <int>();
foreach (long Block in bla)
{
BlockI0.Add(m_op.Mapping.GetBlockI0((int)Block));
Block_N.Add(m_op.Mapping.GetBlockLen((int)Block));
}
BlockI0.SaveToTextFileDebug("BlockI0");
Block_N.SaveToTextFileDebug("Block_N");
#endif
}