/// <summary>
/// returns all external rows of <paramref name="M"/>
/// corresponding to ghost cells of <paramref name="map"/>,
/// which are located on other Mpi-ranks.
/// </summary>
/// <param name="map">Multigrid mapping</param>
/// <param name="M">matrix distributed according to <paramref name="map"/></param>
/// <returns></returns>
public static BlockMsrMatrix GetAllExternalRows(MultigridMapping map, BlockMsrMatrix M)
{
var extcells = map.AggGrid.iLogicalCells.NoOfExternalCells.ForLoop(i => i + map.LocalNoOfBlocks);
var SBS = new SubBlockSelector(map);
SBS.CellSelector(extcells, false);
var AllExtMask = new BlockMaskExt(SBS, 0);
var ExternalRows_BlockI0 = AllExtMask.GetAllSubMatrixCellOffsets();
var ExternalRows_BlockN = AllExtMask.GetAllSubMatrixCellLength();
var ExternalRowsIndices = AllExtMask.m_GlobalMask;
BlockPartitioning PermRow = new BlockPartitioning(ExternalRowsIndices.Count, ExternalRows_BlockI0, ExternalRows_BlockN, M.MPI_Comm, i0isLocal: true);
BlockMsrMatrix Perm = new BlockMsrMatrix(PermRow, M._RowPartitioning);
for (int iRow = 0; iRow < ExternalRowsIndices.Count; iRow++)
{
Debug.Assert(M._RowPartitioning.IsInLocalRange(ExternalRowsIndices[iRow]) == false);
Perm[iRow + PermRow.i0, ExternalRowsIndices[iRow]] = 1;
}
#if TEST
Perm.SaveToTextFileSparseDebug("Perm");
#endif
return(BlockMsrMatrix.Multiply(Perm, M));
}
public static int GetLocalandExternalDOF(MultigridMapping map)
{
int eCell = map.LocalNoOfBlocks + map.AggGrid.iLogicalCells.NoOfExternalCells - 1;
int eVar = map.AggBasis.Length - 1;
int eN = map.AggBasis[eVar].GetLength(eCell, map.DgDegree[eVar]) - 1;
return(map.LocalUniqueIndex(eVar, eCell, eN) + 1);
}
/// <summary>
/// Specifies, which blocks in a matrix shall be selected. Blocksubdivision Default: Selects all blocks.
/// </summary>
/// <param name="map"></param>
public SubBlockSelectorBase(MultigridMapping map)
{
if (map == null)
{
throw new ArgumentNullException("empty mapping! This will not end well ...");
}
m_map = map;
this.CellSelector();
this.VariableSelector();
this.SpeciesSelector();
this.ModeSelector();
}
/// <summary>
/// Prolongation or Restriction from *any* other level to this level.
/// </summary>
/// <param name="otherLevel">
/// Other level, from which one wants to prolongate/restrict.
/// </param>
/// <returns>
/// A matrix, where
/// - row correspond to this mapping
/// - columns correspond to <paramref name="otherLevel"/>
/// If the other level is coarser, this is a prolongation; if the other level is finer, it is the restriction in the L2-sense,
/// for standard DG; for XDG, in some other norm determined by the cut-cell shape.
/// </returns>
public BlockMsrMatrix FromOtherLevelMatrix(MultigridMapping otherLevel)
{
using (new FuncTrace()) {
BlockMsrMatrix PrlgMtx;
{
PrlgMtx = new BlockMsrMatrix(otherLevel, otherLevel.ProblemMapping);
for (int ifld = 0; ifld < otherLevel.AggBasis.Length; ifld++)
{
otherLevel.AggBasis[ifld].GetRestrictionMatrix(PrlgMtx, otherLevel, ifld); // prolongate from the other level to the full grid
}
PrlgMtx = PrlgMtx.Transpose();
}
BlockMsrMatrix RestMtx;
{
RestMtx = new BlockMsrMatrix(this, this.ProblemMapping);
for (int ifld = 0; ifld < this.AggBasis.Length; ifld++)
{
this.AggBasis[ifld].GetRestrictionMatrix(RestMtx, this, ifld); // ... and restrict to this level
}
}
var result = BlockMsrMatrix.Multiply(RestMtx, PrlgMtx);
#if DEBUG
{
var resultT = result.Transpose();
BlockMsrMatrix ShoudBeId;
if (result.RowPartitioning.TotalLength < result.ColPartition.TotalLength)
{
ShoudBeId = BlockMsrMatrix.Multiply(result, resultT);
}
else
{
ShoudBeId = BlockMsrMatrix.Multiply(resultT, result);
}
ShoudBeId.AccEyeSp(-1.0);
double ShouldBeID_Norm = ShoudBeId.InfNorm();
Debug.Assert(ShouldBeID_Norm < 1.0e-8);
//Console.WriteLine("Id norm {0} \t (level {1})", ShouldBeID_Norm, this.AggGrid.MgLevel);
}
#endif
return(result);
}
}
public override void GetRestrictionMatrix(BlockMsrMatrix RST, MultigridMapping mgMap, int iF)
{
if (!object.ReferenceEquals(mgMap.AggBasis[iF], this))
{
throw new ArgumentException();
}
//MsrMatrix RST = new MsrMatrix(mgMap.Partitioning, mgMap.ProblemMapping);
int JAGG = this.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
int[] degrees = mgMap.DgDegree;
int NoFld = degrees.Length;
int N_rest; // = new int[NoFld];
int N_full; // = new int[NoFld];
{
BoSSS.Foundation.Basis b = mgMap.ProblemMapping.BasisS[iF];
BoSSS.Foundation.Basis NxB = (b is BoSSS.Foundation.XDG.XDGBasis) ? ((BoSSS.Foundation.XDG.XDGBasis)b).NonX_Basis : b;
N_rest = NxB.Polynomials[0].Where(poly => poly.AbsoluteDegree <= degrees[iF]).Count();
N_full = NxB.Length;
}
int mgMap_Offset = mgMap.Partitioning.i0;
for (int jAgg = 0; jAgg < JAGG; jAgg++)
{
int[] AgCell = this.AggGrid.iLogicalCells.AggregateCellToParts[jAgg];
int K = AgCell.Length;
int NoSpc_Agg = this.NoOfSpecies[jAgg]; // number of species in aggregate cell
for (int iSpc_Agg = 0; iSpc_Agg < NoSpc_Agg; iSpc_Agg++) // loop over all species in aggregate cell
{
{ // loop over DG fields in the mapping
int NROW = N_rest;
int NCOL = N_full;
Debug.Assert(mgMap.AggBasis[iF].GetLength(jAgg, degrees[iF]) == N_rest * NoSpc_Agg);
int i0Agg_Loc = mgMap.LocalUniqueIndex(iF, jAgg, N_rest * iSpc_Agg);
int i0Agg = i0Agg_Loc + mgMap_Offset;
Debug.Assert(i0Agg >= mgMap.Partitioning.i0);
Debug.Assert(i0Agg < mgMap.Partitioning.iE);
if (this.SpeciesIndexMapping[jAgg] == null)
{
Debug.Assert(NoSpc_Agg == 1);
Debug.Assert(NoSpc_Agg == 1);
// default branch:
// use restriction/prolongation from un-cut basis
// +++++++++++++++++++++++++++++++++++++++++++++
MultidimensionalArray Trf = base.CompositeBasis[jAgg];
for (int k = 0; k < K; k++) // loop over the cells which form the aggregated cell...
{
int jCell = AgCell[k];
int i0Full = mgMap.ProblemMapping.GlobalUniqueCoordinateIndex(iF, jCell, 0);
var Block = Trf.ExtractSubArrayShallow(k, -1, -1);
for (int nRow = 0; nRow < NROW; nRow++)
{
for (int nCol = 0; nCol < NCOL; nCol++)
{
RST[i0Agg + nRow, i0Full + nCol] = Block[nCol, nRow];
}
}
}
}
else
{
int NoSpc = this.NoOfSpecies[jAgg];
int[,] sim = SpeciesIndexMapping[jAgg];
Debug.Assert(sim.GetLength(0) == NoSpc);
Debug.Assert(sim.GetLength(1) == K);
MultidimensionalArray Trf;
if (this.XCompositeBasis[jAgg] == null || this.XCompositeBasis[jAgg][iSpc_Agg] == null)
{
Trf = base.CompositeBasis[jAgg];
}
else
{
Trf = this.XCompositeBasis[jAgg][iSpc_Agg];
}
for (int k = 0; k < K; k++) // loop over the cells which form the aggregated cell...
{
int jCell = AgCell[k];
int iSpcBase = sim[iSpc_Agg, k];
if (iSpcBase < 0)
{
//for(int n = 0; n < N; n++)
// FulCoords[k, n] = 0;
}
else
{
int i0Full = mgMap.ProblemMapping.GlobalUniqueCoordinateIndex(iF, jCell, iSpcBase * N_full);
var Block = Trf.ExtractSubArrayShallow(k, -1, -1);
for (int nRow = 0; nRow < NROW; nRow++)
{
for (int nCol = 0; nCol < NCOL; nCol++)
{
RST[i0Agg + nRow, i0Full + nCol] = Block[nCol, nRow];
}
}
}
}
}
}
}
}
//return RST;
}
/// <summary>
/// ctor.
/// </summary>
public MPIexchange(MultigridMapping map, T vector)
{
// misc init
// =========
IGridData master = map.AggGrid;
int J = master.iLogicalCells.Count;
if (vector.Count != map.LocalLength)
{
throw new ArgumentException("wrong length of input vector.");
}
m_vector = vector;
m_master = master;
m_map = map;
var Para = m_master.iParallel;
var rvcProc = Para.ProcessesToReceiveFrom;
var sndProc = Para.ProcessesToSendTo;
rqst = new MPI_Request[sndProc.Length + rvcProc.Length];
// allocate send buffers
// =====================
{
SendBuffers = new double[sndProc.Length][];
for (int i = 0; i < SendBuffers.Length; i++)
{
int p = sndProc[i];
// compute length of send list
int L = 0;
foreach (int jCell in Para.SendCommLists[p])
{
Debug.Assert(map.IsLocalBlock(jCell + map.FirstBlock));
Debug.Assert(map.GetLength(jCell) == map.GetBlockLen(jCell + map.FirstBlock));
L += map.GetLength(jCell);
}
// alloc send buffer
SendBuffers[i] = new double[L];
}
SendBufferPin = new GCHandle[sndProc.Length];
}
// allocate receive buffers
// ========================
{
int totL = 0;
RcvBuffer = new double[rvcProc.Length][];
for (int i = 0; i < RcvBuffer.Length; i++)
{
int p = rvcProc[i];
// compute length of receive list
int L = 0;
int J0 = Para.RcvCommListsInsertIndex[p];
int JE = Para.RcvCommListsNoOfItems[p] + J0;
for (int jCell = J0; jCell < JE; jCell++)
{
Debug.Assert(jCell >= map.LocalNoOfBlocks);
L += map.GetLength(jCell);
}
totL += L;
// alloc internal receive buffer
RcvBuffer[i] = new double[L];
}
RcvBufferPin = new GCHandle[RcvBuffer.Length];
m_Vector_Ext = new double[totL];
}
}
/// <summary>
/// ctor.
/// </summary>
public MPIexchange(MultigridMapping map, T vector)
{
// misc init
// =========
IGridData master = map.AggGrid;
int J = master.iLogicalCells.Count;
if (vector.Count != map.LocalLength)
{
throw new ArgumentException("wrong length of input vector.");
}
m_vector = vector;
m_master = master;
m_map = map;
var Para = m_master.iParallel;
var rvcProc = Para.ProcessesToReceiveFrom;
var sndProc = Para.ProcessesToSendTo;
rqst = new MPI_Request[sndProc.Length + rvcProc.Length];
// allocate send buffers
// =====================
{
SendBuffers = new double[sndProc.Length][];
for (int i = 0; i < SendBuffers.Length; i++)
{
int p = sndProc[i];
// compute length of send list
int L = 0;
foreach (int jCell in Para.SendCommLists[p])
{
Debug.Assert(map.IsLocalBlock(jCell + map.FirstBlock));
Debug.Assert(map.GetLength(jCell) == map.GetBlockLen(jCell + map.FirstBlock));
L += map.GetLength(jCell);
}
// alloc send buffer
SendBuffers[i] = new double[L];
}
SendBufferPin = new GCHandle[sndProc.Length];
}
// allocate receive buffers
// ========================
{
int totL = 0;
RcvBuffer = new double[rvcProc.Length][];
for (int i = 0; i < RcvBuffer.Length; i++)
{
int p = rvcProc[i];
// compute length of receive list
int L = 0;
int J0 = Para.RcvCommListsInsertIndex[p];
int JE = Para.RcvCommListsNoOfItems[p] + J0;
for (int jCell = J0; jCell < JE; jCell++)
{
Debug.Assert(jCell >= map.LocalNoOfBlocks);
L += map.GetLength(jCell);
}
totL += L;
// alloc internal receive buffer
RcvBuffer[i] = new double[L];
}
RcvBufferPin = new GCHandle[RcvBuffer.Length];
m_Vector_Ext = new double[totL];
}
#if DEBUG
// verify send and receive lengths
// ===============================
{
var RcvSizes = new Dictionary <int, int[]>();
for (int i = 0; i < SendBuffers.Length; i++)
{
//Console.WriteLine("P{0}: to proc {1}: {2} items ", map.MpiRank, sndProc[i], SendBuffers[i].Length);
int source = map.MpiRank;
int target = sndProc[i];
int L = SendBuffers[i].Length;
RcvSizes.Add(target, new[] { source, L });
}
var _RcvSizes = ilPSP.Utils.SerialisationMessenger.ExchangeData(RcvSizes);
foreach (var kv in _RcvSizes)
{
int source = kv.Key;
int sourceR = kv.Value[0];
int L = kv.Value[1];
int idxSrs = Array.IndexOf(rvcProc, source);
Debug.Assert(idxSrs >= 0);
int _L = RcvBuffer[idxSrs].Length;
Debug.Assert(source == sourceR);
Debug.Assert(L == _L, "mismatch in receive buffer size on multigrid level" + map.AggGrid.MgLevel);
}
csMPI.Raw.Barrier(map.MPI_Comm);
}
#endif
}
/// <summary>
/// Prolongation/Injection operator to finer grid level.
/// </summary>
public BlockMsrMatrix GetProlongationOperator(MultigridMapping finerLevel)
{
using (new FuncTrace()) {
// Argument checking
// =================
if (!object.ReferenceEquals(finerLevel.AggGrid, this.AggGrid.ParentGrid))
{
throw new ArgumentException("Only prolongation/injection to next level is supported.");
}
if (finerLevel.AggBasis.Length != this.AggBasis.Length)
{
throw new ArgumentException("");
}
int NoOfVar = this.AggBasis.Length;
MultidimensionalArray[][] InjOp = new MultidimensionalArray[NoOfVar][];
AggregationGridBasis[] B = new AggregationGridBasis[NoOfVar];
bool[] useX = new bool[NoOfVar];
int[] DegreeS = new int[NoOfVar];
int[] DegreeSfine = new int[NoOfVar];
for (int iVar = 0; iVar < NoOfVar; iVar++)
{
InjOp[iVar] = this.AggBasis[iVar].InjectionOperator;
B[iVar] = AggBasis[iVar];
DegreeS[iVar] = this.DgDegree[iVar];
DegreeSfine[iVar] = finerLevel.DgDegree[iVar];
if (DegreeSfine[iVar] < DegreeS[iVar])
{
throw new ArgumentException("Lower DG degree on finer grid is not supported by this method ");
}
useX[iVar] = this.AggBasis[iVar] is XdgAggregationBasis;
if (useX[iVar] != (finerLevel.AggBasis[iVar] is XdgAggregationBasis))
{
throw new ArgumentException("XDG / DG mismatch between this and finer level for " + iVar + "-th variable.");
}
}
XdgAggregationBasis XB = null;
XdgAggregationBasis XBf = null;
int[][,] spcIdxMap = null;
SpeciesId[][] spc = null;
//SpeciesId[][] spcf = null;
for (int iVar = 0; iVar < NoOfVar; iVar++)
{
if (useX[iVar])
{
XB = (XdgAggregationBasis)(B[iVar]);
XBf = (XdgAggregationBasis)(finerLevel.AggBasis[iVar]);
spcIdxMap = XB.SpeciesIndexMapping;
spc = XB.AggCellsSpecies;
//spcf = XBf.AggCellsSpecies;
break;
}
}
int[] Np = this.AggBasis[0].GetNp();
int[] Np_fine = finerLevel.AggBasis[0].GetNp();
// create matrix
// =============
// init retval
var PrlgMtx = new BlockMsrMatrix(finerLevel, this);
int[][] C2F = this.AggGrid.jCellCoarse2jCellFine;
int JCoarse = this.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
//Debug.Assert((JCoarse == C2F.Length) || ());
for (int jc = 0; jc < JCoarse; jc++) // loop over coarse cells...
{
int[] AggCell = C2F[jc];
int I = AggCell.Length;
for (int iVar = 0; iVar < NoOfVar; iVar++)
{
int DgDeg = DegreeS[iVar];
int DgDegF = DegreeSfine[iVar];
MultidimensionalArray Inj_iVar_jc = InjOp[iVar][jc];
Debug.Assert(Inj_iVar_jc.GetLength(0) == I);
bool useX_iVar = false;
if (useX[iVar])
{
if (spcIdxMap[jc] != null)
{
useX_iVar = true;
}
}
if (useX_iVar)
{
//throw new NotImplementedException("todo");
int NoOfSpc = XB.GetNoOfSpecies(jc);
int Np_col = Np[DgDeg];
Debug.Assert(Np_col * NoOfSpc == B[iVar].GetLength(jc, DgDeg));
for (int iSpc = 0; iSpc < NoOfSpc; iSpc++) // loop over species
{
SpeciesId spc_jc_i = spc[jc][iSpc];
int Col0 = this.GlobalUniqueIndex(iVar, jc, Np_col * iSpc);
for (int i = 0; i < I; i++) // loop over finer cells
{
int jf = AggCell[i];
int iSpc_Row = XBf.GetSpeciesIndex(jf, spc_jc_i);
if (iSpc_Row < 0)
{
// nothing to do
continue;
}
int Np_row = Np_fine[DgDegF];
Debug.Assert(Np_row * XBf.GetNoOfSpecies(jf) == finerLevel.AggBasis[iVar].GetLength(jf, DgDegF));
int Row0 = finerLevel.GlobalUniqueIndex(iVar, jf, Np_row * iSpc_Row);
//if(Row0 <= 12 && 12 < Row0 + Np_row) {
// if(Col0 <= 3 && 3 < Col0 + Np_col) {
// Debugger.Break();
// }
//}
PrlgMtx.AccBlock(Row0, Col0, 1.0, Inj_iVar_jc.ExtractSubArrayShallow(new[] { i, 0, 0 }, new[] { i - 1, Np_row - 1, Np_col - 1 }));
}
}
}
else
{
// ++++++++++++++++++
// standard DG branch
// ++++++++++++++++++
int Np_col = Np[DgDeg];
Debug.Assert(Np_col == B[iVar].GetLength(jc, DgDeg));
int Col0 = this.GlobalUniqueIndex(iVar, jc, 0);
for (int i = 0; i < I; i++) // loop over finer cells
{
int jf = AggCell[i];
int Np_row = Np_fine[DgDegF];
Debug.Assert(Np_row == finerLevel.AggBasis[iVar].GetLength(jf, DgDegF));
int Row0 = finerLevel.GlobalUniqueIndex(iVar, jf, 0);
PrlgMtx.AccBlock(Row0, Col0, 1.0, Inj_iVar_jc.ExtractSubArrayShallow(new[] { i, 0, 0 }, new[] { i - 1, Np_row - 1, Np_col - 1 }));
//if(Row0 <= 12 && 12 < Row0 + Np_row) {
// if(Col0 <= 3 && 3 < Col0 + Np_col) {
// Debugger.Break();
// }
// }
}
}
}
}
// return
// ======
return(PrlgMtx);
}
}
public SubBlockSelector(MultigridMapping map) : base(map)
{
}