/// <summary>
/// initiates the send/receive - processes and returns immediately;
/// Every call of this method must be matched by a later call to <see cref="TransceiveFinish"/>;
/// </summary>
public void TransceiveStartImReturn()
{
ilPSP.MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
var Para = m_master.iParallel;
int[] sndProc = Para.ProcessesToSendTo;
int MyRank;
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out MyRank);
Array.Clear(this.rqst, 0, this.rqst.Length);
unsafe {
// Sending ...
// -----------
// over all processes to which we have to send data to ...
for (int i = 0; i < sndProc.Length; i++)
{
// destination processor and comm list
int pDest = sndProc[i];
int[] commList = Para.SendCommLists[pDest];
int Len = commList.Length;
// fill send buffer
var SendBuffer = SendBuffers[i];
int cnt = 0;
for (int l = 0; l < Len; l++)
{
int jCell = commList[l];
int N = m_map.GetLength(jCell);
int i0 = m_map.LocalUniqueIndex(0, jCell, 0);
for (int n = 0; n < N; n++)
{
SendBuffer[cnt] = this.m_vector[i0 + n];
cnt++;
}
}
Debug.Assert(cnt == SendBuffers[i].Length);
// MPI send
SendBufferPin[i] = GCHandle.Alloc(SendBuffers[i], GCHandleType.Pinned);
csMPI.Raw.Issend(Marshal.UnsafeAddrOfPinnedArrayElement(SendBuffers[i], 0),
SendBuffers[i].Length, csMPI.Raw._DATATYPE.DOUBLE, pDest,
4442 + MyRank,
csMPI.Raw._COMM.WORLD,
out rqst[i]);
}
}
}
public static int GetLocalandExternalDOF(MultigridMapping map)
{
int eCell = map.LocalNoOfBlocks + map.AggGrid.iLogicalCells.NoOfExternalCells - 1;
int eVar = map.AggBasis.Length - 1;
int eN = map.AggBasis[eVar].GetLength(eCell, map.DgDegree[eVar]) - 1;
return(map.LocalUniqueIndex(eVar, eCell, eN) + 1);
}
public override void GetRestrictionMatrix(BlockMsrMatrix RST, MultigridMapping mgMap, int iF)
{
if (!object.ReferenceEquals(mgMap.AggBasis[iF], this))
{
throw new ArgumentException();
}
//MsrMatrix RST = new MsrMatrix(mgMap.Partitioning, mgMap.ProblemMapping);
int JAGG = this.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
int[] degrees = mgMap.DgDegree;
int NoFld = degrees.Length;
int N_rest; // = new int[NoFld];
int N_full; // = new int[NoFld];
{
BoSSS.Foundation.Basis b = mgMap.ProblemMapping.BasisS[iF];
BoSSS.Foundation.Basis NxB = (b is BoSSS.Foundation.XDG.XDGBasis) ? ((BoSSS.Foundation.XDG.XDGBasis)b).NonX_Basis : b;
N_rest = NxB.Polynomials[0].Where(poly => poly.AbsoluteDegree <= degrees[iF]).Count();
N_full = NxB.Length;
}
int mgMap_Offset = mgMap.Partitioning.i0;
for (int jAgg = 0; jAgg < JAGG; jAgg++)
{
int[] AgCell = this.AggGrid.iLogicalCells.AggregateCellToParts[jAgg];
int K = AgCell.Length;
int NoSpc_Agg = this.NoOfSpecies[jAgg]; // number of species in aggregate cell
for (int iSpc_Agg = 0; iSpc_Agg < NoSpc_Agg; iSpc_Agg++) // loop over all species in aggregate cell
{
{ // loop over DG fields in the mapping
int NROW = N_rest;
int NCOL = N_full;
Debug.Assert(mgMap.AggBasis[iF].GetLength(jAgg, degrees[iF]) == N_rest * NoSpc_Agg);
int i0Agg_Loc = mgMap.LocalUniqueIndex(iF, jAgg, N_rest * iSpc_Agg);
int i0Agg = i0Agg_Loc + mgMap_Offset;
Debug.Assert(i0Agg >= mgMap.Partitioning.i0);
Debug.Assert(i0Agg < mgMap.Partitioning.iE);
if (this.SpeciesIndexMapping[jAgg] == null)
{
Debug.Assert(NoSpc_Agg == 1);
Debug.Assert(NoSpc_Agg == 1);
// default branch:
// use restriction/prolongation from un-cut basis
// +++++++++++++++++++++++++++++++++++++++++++++
MultidimensionalArray Trf = base.CompositeBasis[jAgg];
for (int k = 0; k < K; k++) // loop over the cells which form the aggregated cell...
{
int jCell = AgCell[k];
int i0Full = mgMap.ProblemMapping.GlobalUniqueCoordinateIndex(iF, jCell, 0);
var Block = Trf.ExtractSubArrayShallow(k, -1, -1);
for (int nRow = 0; nRow < NROW; nRow++)
{
for (int nCol = 0; nCol < NCOL; nCol++)
{
RST[i0Agg + nRow, i0Full + nCol] = Block[nCol, nRow];
}
}
}
}
else
{
int NoSpc = this.NoOfSpecies[jAgg];
int[,] sim = SpeciesIndexMapping[jAgg];
Debug.Assert(sim.GetLength(0) == NoSpc);
Debug.Assert(sim.GetLength(1) == K);
MultidimensionalArray Trf;
if (this.XCompositeBasis[jAgg] == null || this.XCompositeBasis[jAgg][iSpc_Agg] == null)
{
Trf = base.CompositeBasis[jAgg];
}
else
{
Trf = this.XCompositeBasis[jAgg][iSpc_Agg];
}
for (int k = 0; k < K; k++) // loop over the cells which form the aggregated cell...
{
int jCell = AgCell[k];
int iSpcBase = sim[iSpc_Agg, k];
if (iSpcBase < 0)
{
//for(int n = 0; n < N; n++)
// FulCoords[k, n] = 0;
}
else
{
int i0Full = mgMap.ProblemMapping.GlobalUniqueCoordinateIndex(iF, jCell, iSpcBase * N_full);
var Block = Trf.ExtractSubArrayShallow(k, -1, -1);
for (int nRow = 0; nRow < NROW; nRow++)
{
for (int nCol = 0; nCol < NCOL; nCol++)
{
RST[i0Agg + nRow, i0Full + nCol] = Block[nCol, nRow];
}
}
}
}
}
}
}
}
//return RST;
}
/// <summary>
/// initiates the send/receive - processes and returns immediately;
/// Every call of this method must be matched by a later call to <see cref="TransceiveFinish"/>;
/// </summary>
public void TransceiveStartImReturn()
{
ilPSP.MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
var Para = m_master.iParallel;
int[] sndProc = Para.ProcessesToReceiveFrom; // yes, intentional
int MyRank;
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out MyRank);
Array.Clear(this.rqst, 0, this.rqst.Length);
unsafe {
// Sending ...
// -----------
// over all processes to which we have to send data to ...
for (int i = 0; i < sndProc.Length; i++)
{
// destination processor and external cell range
int pDest = sndProc[i];
int J0 = Para.RcvCommListsInsertIndex[pDest];
int JE = Para.RcvCommListsNoOfItems[pDest] + J0;
// fill send buffer
var SendBuffer = SendBuffers[i];
int extOffset = m_map.LocalLength;
int cnt = 0;
for (int jCell = J0; jCell < JE; jCell++)
{
Debug.Assert(jCell >= m_master.iLogicalCells.NoOfLocalUpdatedCells);
int N = m_map.GetLength(jCell);
int i0 = m_map.LocalUniqueIndex(0, jCell, 0);
for (int n = 0; n < N; n++)
{
SendBuffer[cnt] = this.m_Vector_Ext[i0 + n - extOffset];
cnt++;
}
}
Debug.Assert(cnt == SendBuffers[i].Length);
// MPI send
SendBufferPin[i] = GCHandle.Alloc(SendBuffers[i], GCHandleType.Pinned);
csMPI.Raw.Issend(Marshal.UnsafeAddrOfPinnedArrayElement(SendBuffers[i], 0),
SendBuffers[i].Length, csMPI.Raw._DATATYPE.DOUBLE, pDest,
4442 + MyRank,
csMPI.Raw._COMM.WORLD,
out rqst[i]);
}
}
}
/// <summary>
/// The core of the masking
/// Generates index lists and the Ni0-struct-list corresponding to mask
/// and is called by the child classes: local and external mask
/// Note: the smallest unit are DG sub blocks!
/// </summary>
protected void GenerateAllMasks()
{
int NoOfCells = m_NoOfCells;
int NoOfVariables = m_NoOfVariables;
int[][] NoOfSpecies = m_NoOfSpecies;
int[] DGdegreeP1 = m_DGdegree.CloneAs();
for (int iDG = 0; iDG < DGdegreeP1.Length; iDG++)
{
DGdegreeP1[iDG] += 1;
}
List <extNi0> ListNi0 = new List <extNi0>();
List <int> Globalint = new List <int>();
List <int> Localint = new List <int>();
List <int> SubBlockIdx = new List <int>();
int SubOffset = m_SubBlockOffset; // 0 for local mask and BMLoc.LocalDof for external mask
int Ni0Length = 0;
int MaskLen = 0;
int prevLocie = m_map.LocalNoOfBlocks;
var tmpCell = new List <extNi0[][][]>();
// local caching of filter functions
// ensures that functions are not re-allocated during the loops
var CellInstruction = m_sbs.CellFilter;
var VarInstruction = m_sbs.VariableFilter;
var SpecInstruction = m_sbs.SpeciesFilter;
var ModeInstruction = m_sbs.ModeFilter;
bool emptysel = true;
// loop over cells...
for (int iLoc = 0; iLoc < NoOfCells; iLoc++)
{
int jLoc = m_CellOffset + iLoc; //to address correctly, external cells offset has to be concidered, you know ...
emptysel &= !CellInstruction(jLoc); //for testing if the entire selection is empty, which hopefully only can happen at the level of cells
if (!CellInstruction(jLoc))
{
continue;
}
var tmpVar = new List <extNi0[][]>();
// loop over variables...
for (int iVar = 0; iVar < NoOfVariables; iVar++)
{
if (!VarInstruction(jLoc, iVar))
{
continue;
}
var tmpSpc = new List <extNi0[]>();
// loop over species...
for (int iSpc = 0; iSpc < NoOfSpecies[iLoc][iVar]; iSpc++)
{
if (!SpecInstruction(jLoc, iVar, iSpc))
{
continue;
}
int GlobalOffset = m_map.GlobalUniqueIndex(iVar, jLoc, iSpc, 0);
int LocalOffset = m_map.LocalUniqueIndex(iVar, jLoc, iSpc, 0);
var tmpMod = new List <extNi0>();
// loop over polynomial degrees...
for (int degree = 0; degree < DGdegreeP1[iVar]; degree++)
{
if (ModeInstruction(jLoc, iVar, iSpc, degree))
{
int GlobalModeOffset = m_Ni0[degree].i0 + GlobalOffset;
int LocalModeOffset = m_Ni0[degree].i0 + LocalOffset;
int ModeLength = m_Ni0[degree].N;
var newNi0 = new extNi0(LocalModeOffset, GlobalModeOffset, SubOffset, ModeLength);
SubOffset += ModeLength;
// Fill int lists
for (int i = 0; i < newNi0.N; i++)
{
Globalint.Add(newNi0.Gi0 + i);
Localint.Add(newNi0.Li0 + i);
SubBlockIdx.Add(newNi0.Si0 + i);
MaskLen++;
}
// Fill Ni0 Lists
tmpMod.Add(newNi0);
Ni0Length++;
ListNi0.Add(newNi0);
Debug.Assert(m_map.LocalUniqueIndex(iVar, jLoc, iSpc, GetNp(degree) - 1) == LocalModeOffset + ModeLength - 1);
}
}
if (tmpMod.Count > 0)
{
tmpSpc.Add(tmpMod.ToArray());
}
}
if (tmpSpc.Count > 0)
{
tmpVar.Add(tmpSpc.ToArray());
}
}
if (tmpVar.Count > 0)
{
tmpCell.Add(tmpVar.ToArray());
}
}
var tmpStructNi0 = tmpCell.ToArray();
int NumOfNi0 = 0;
#if DEBUG
for (int iCell = 0; iCell < tmpStructNi0.Length; iCell++)
{
for (int iVar = 0; iVar < tmpStructNi0[iCell].Length; iVar++)
{
for (int iSpc = 0; iSpc < tmpStructNi0[iCell][iVar].Length; iSpc++)
{
NumOfNi0 += tmpStructNi0[iCell][iVar][iSpc].Length;
}
}
}
#endif
// an empty selection is allowed,
// e.g. consider a combination of empty external and non empty local mask
if (!emptysel)
{
Debug.Assert(ListNi0.GroupBy(x => x.Li0).Any(g => g.Count() == 1));
Debug.Assert(ListNi0.GroupBy(x => x.Gi0).Any(g => g.Count() == 1));
Debug.Assert(ListNi0.Count() == NumOfNi0);
Debug.Assert(MaskLen <= m_LocalLength);
Debug.Assert(Localint.GroupBy(x => x).Any(g => g.Count() == 1));
Debug.Assert(Globalint.GroupBy(x => x).Any(g => g.Count() == 1));
Debug.Assert(Localint.Count() == MaskLen);
Debug.Assert(SubBlockIdx.Count() == MaskLen);
Debug.Assert(SubBlockIdx.GroupBy(x => x).Any(g => g.Count() == 1));
}
m_GlobalMask = Globalint;
m_LocalMask = Localint;
m_StructuredNi0 = tmpStructNi0;
m_MaskLen = MaskLen;
m_Ni0Len = Ni0Length;
m_SubBlockMask = SubBlockIdx;
}
