/// <summary>
/// initiates the send/receive - processes and returns immediately;
/// Every call of this method must be matched by a later call to <see cref="TransceiveFinish"/>;
/// </summary>
public void TransceiveStartImReturn()
{
ilPSP.MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
var Para = m_master.iParallel;
int[] sndProc = Para.ProcessesToSendTo;
int MyRank;
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out MyRank);
Array.Clear(this.rqst, 0, this.rqst.Length);
unsafe {
// Sending ...
// -----------
// over all processes to which we have to send data to ...
for (int i = 0; i < sndProc.Length; i++)
{
// destination processor and comm list
int pDest = sndProc[i];
int[] commList = Para.SendCommLists[pDest];
int Len = commList.Length;
// fill send buffer
var SendBuffer = SendBuffers[i];
int cnt = 0;
for (int l = 0; l < Len; l++)
{
int jCell = commList[l];
int N = m_map.GetLength(jCell);
int i0 = m_map.LocalUniqueIndex(0, jCell, 0);
for (int n = 0; n < N; n++)
{
SendBuffer[cnt] = this.m_vector[i0 + n];
cnt++;
}
}
Debug.Assert(cnt == SendBuffers[i].Length);
// MPI send
SendBufferPin[i] = GCHandle.Alloc(SendBuffers[i], GCHandleType.Pinned);
csMPI.Raw.Issend(Marshal.UnsafeAddrOfPinnedArrayElement(SendBuffers[i], 0),
SendBuffers[i].Length, csMPI.Raw._DATATYPE.DOUBLE, pDest,
4442 + MyRank,
csMPI.Raw._COMM.WORLD,
out rqst[i]);
}
}
}
public override void GetRestrictionMatrix(BlockMsrMatrix RST, MultigridMapping mgMap, int iF)
{
if (!object.ReferenceEquals(mgMap.AggBasis[iF], this))
{
throw new ArgumentException();
}
//MsrMatrix RST = new MsrMatrix(mgMap.Partitioning, mgMap.ProblemMapping);
int JAGG = this.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
int[] degrees = mgMap.DgDegree;
int NoFld = degrees.Length;
int N_rest; // = new int[NoFld];
int N_full; // = new int[NoFld];
{
BoSSS.Foundation.Basis b = mgMap.ProblemMapping.BasisS[iF];
BoSSS.Foundation.Basis NxB = (b is BoSSS.Foundation.XDG.XDGBasis) ? ((BoSSS.Foundation.XDG.XDGBasis)b).NonX_Basis : b;
N_rest = NxB.Polynomials[0].Where(poly => poly.AbsoluteDegree <= degrees[iF]).Count();
N_full = NxB.Length;
}
int mgMap_Offset = mgMap.Partitioning.i0;
for (int jAgg = 0; jAgg < JAGG; jAgg++)
{
int[] AgCell = this.AggGrid.iLogicalCells.AggregateCellToParts[jAgg];
int K = AgCell.Length;
int NoSpc_Agg = this.NoOfSpecies[jAgg]; // number of species in aggregate cell
for (int iSpc_Agg = 0; iSpc_Agg < NoSpc_Agg; iSpc_Agg++) // loop over all species in aggregate cell
{
{ // loop over DG fields in the mapping
int NROW = N_rest;
int NCOL = N_full;
Debug.Assert(mgMap.AggBasis[iF].GetLength(jAgg, degrees[iF]) == N_rest * NoSpc_Agg);
int i0Agg_Loc = mgMap.LocalUniqueIndex(iF, jAgg, N_rest * iSpc_Agg);
int i0Agg = i0Agg_Loc + mgMap_Offset;
Debug.Assert(i0Agg >= mgMap.Partitioning.i0);
Debug.Assert(i0Agg < mgMap.Partitioning.iE);
if (this.SpeciesIndexMapping[jAgg] == null)
{
Debug.Assert(NoSpc_Agg == 1);
Debug.Assert(NoSpc_Agg == 1);
// default branch:
// use restriction/prolongation from un-cut basis
// +++++++++++++++++++++++++++++++++++++++++++++
MultidimensionalArray Trf = base.CompositeBasis[jAgg];
for (int k = 0; k < K; k++) // loop over the cells which form the aggregated cell...
{
int jCell = AgCell[k];
int i0Full = mgMap.ProblemMapping.GlobalUniqueCoordinateIndex(iF, jCell, 0);
var Block = Trf.ExtractSubArrayShallow(k, -1, -1);
for (int nRow = 0; nRow < NROW; nRow++)
{
for (int nCol = 0; nCol < NCOL; nCol++)
{
RST[i0Agg + nRow, i0Full + nCol] = Block[nCol, nRow];
}
}
}
}
else
{
int NoSpc = this.NoOfSpecies[jAgg];
int[,] sim = SpeciesIndexMapping[jAgg];
Debug.Assert(sim.GetLength(0) == NoSpc);
Debug.Assert(sim.GetLength(1) == K);
MultidimensionalArray Trf;
if (this.XCompositeBasis[jAgg] == null || this.XCompositeBasis[jAgg][iSpc_Agg] == null)
{
Trf = base.CompositeBasis[jAgg];
}
else
{
Trf = this.XCompositeBasis[jAgg][iSpc_Agg];
}
for (int k = 0; k < K; k++) // loop over the cells which form the aggregated cell...
{
int jCell = AgCell[k];
int iSpcBase = sim[iSpc_Agg, k];
if (iSpcBase < 0)
{
//for(int n = 0; n < N; n++)
// FulCoords[k, n] = 0;
}
else
{
int i0Full = mgMap.ProblemMapping.GlobalUniqueCoordinateIndex(iF, jCell, iSpcBase * N_full);
var Block = Trf.ExtractSubArrayShallow(k, -1, -1);
for (int nRow = 0; nRow < NROW; nRow++)
{
for (int nCol = 0; nCol < NCOL; nCol++)
{
RST[i0Agg + nRow, i0Full + nCol] = Block[nCol, nRow];
}
}
}
}
}
}
}
}
//return RST;
}
/// <summary>
/// initiates the send/receive - processes and returns immediately;
/// Every call of this method must be matched by a later call to <see cref="TransceiveFinish"/>;
/// </summary>
public void TransceiveStartImReturn()
{
ilPSP.MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
var Para = m_master.iParallel;
int[] sndProc = Para.ProcessesToReceiveFrom; // yes, intentional
int MyRank;
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out MyRank);
Array.Clear(this.rqst, 0, this.rqst.Length);
unsafe {
// Sending ...
// -----------
// over all processes to which we have to send data to ...
for (int i = 0; i < sndProc.Length; i++)
{
// destination processor and external cell range
int pDest = sndProc[i];
int J0 = Para.RcvCommListsInsertIndex[pDest];
int JE = Para.RcvCommListsNoOfItems[pDest] + J0;
// fill send buffer
var SendBuffer = SendBuffers[i];
int extOffset = m_map.LocalLength;
int cnt = 0;
for (int jCell = J0; jCell < JE; jCell++)
{
Debug.Assert(jCell >= m_master.iLogicalCells.NoOfLocalUpdatedCells);
int N = m_map.GetLength(jCell);
int i0 = m_map.LocalUniqueIndex(0, jCell, 0);
for (int n = 0; n < N; n++)
{
SendBuffer[cnt] = this.m_Vector_Ext[i0 + n - extOffset];
cnt++;
}
}
Debug.Assert(cnt == SendBuffers[i].Length);
// MPI send
SendBufferPin[i] = GCHandle.Alloc(SendBuffers[i], GCHandleType.Pinned);
csMPI.Raw.Issend(Marshal.UnsafeAddrOfPinnedArrayElement(SendBuffers[i], 0),
SendBuffers[i].Length, csMPI.Raw._DATATYPE.DOUBLE, pDest,
4442 + MyRank,
csMPI.Raw._COMM.WORLD,
out rqst[i]);
}
}
}