/// <summary> /// initiates the send/receive - processes and returns immediately; /// Every call of this method must be matched by a later call to <see cref="TransceiveFinish"/>; /// </summary> public void TransceiveStartImReturn() { ilPSP.MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD); var Para = m_master.iParallel; int[] sndProc = Para.ProcessesToSendTo; int MyRank; csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out MyRank); Array.Clear(this.rqst, 0, this.rqst.Length); unsafe { // Sending ... // ----------- // over all processes to which we have to send data to ... for (int i = 0; i < sndProc.Length; i++) { // destination processor and comm list int pDest = sndProc[i]; int[] commList = Para.SendCommLists[pDest]; int Len = commList.Length; // fill send buffer var SendBuffer = SendBuffers[i]; int cnt = 0; for (int l = 0; l < Len; l++) { int jCell = commList[l]; int N = m_map.GetLength(jCell); int i0 = m_map.LocalUniqueIndex(0, jCell, 0); for (int n = 0; n < N; n++) { SendBuffer[cnt] = this.m_vector[i0 + n]; cnt++; } } Debug.Assert(cnt == SendBuffers[i].Length); // MPI send SendBufferPin[i] = GCHandle.Alloc(SendBuffers[i], GCHandleType.Pinned); csMPI.Raw.Issend(Marshal.UnsafeAddrOfPinnedArrayElement(SendBuffers[i], 0), SendBuffers[i].Length, csMPI.Raw._DATATYPE.DOUBLE, pDest, 4442 + MyRank, csMPI.Raw._COMM.WORLD, out rqst[i]); } } }
public static int GetLocalandExternalDOF(MultigridMapping map) { int eCell = map.LocalNoOfBlocks + map.AggGrid.iLogicalCells.NoOfExternalCells - 1; int eVar = map.AggBasis.Length - 1; int eN = map.AggBasis[eVar].GetLength(eCell, map.DgDegree[eVar]) - 1; return(map.LocalUniqueIndex(eVar, eCell, eN) + 1); }
public override void GetRestrictionMatrix(BlockMsrMatrix RST, MultigridMapping mgMap, int iF) { if (!object.ReferenceEquals(mgMap.AggBasis[iF], this)) { throw new ArgumentException(); } //MsrMatrix RST = new MsrMatrix(mgMap.Partitioning, mgMap.ProblemMapping); int JAGG = this.AggGrid.iLogicalCells.NoOfLocalUpdatedCells; int[] degrees = mgMap.DgDegree; int NoFld = degrees.Length; int N_rest; // = new int[NoFld]; int N_full; // = new int[NoFld]; { BoSSS.Foundation.Basis b = mgMap.ProblemMapping.BasisS[iF]; BoSSS.Foundation.Basis NxB = (b is BoSSS.Foundation.XDG.XDGBasis) ? ((BoSSS.Foundation.XDG.XDGBasis)b).NonX_Basis : b; N_rest = NxB.Polynomials[0].Where(poly => poly.AbsoluteDegree <= degrees[iF]).Count(); N_full = NxB.Length; } int mgMap_Offset = mgMap.Partitioning.i0; for (int jAgg = 0; jAgg < JAGG; jAgg++) { int[] AgCell = this.AggGrid.iLogicalCells.AggregateCellToParts[jAgg]; int K = AgCell.Length; int NoSpc_Agg = this.NoOfSpecies[jAgg]; // number of species in aggregate cell for (int iSpc_Agg = 0; iSpc_Agg < NoSpc_Agg; iSpc_Agg++) // loop over all species in aggregate cell { { // loop over DG fields in the mapping int NROW = N_rest; int NCOL = N_full; Debug.Assert(mgMap.AggBasis[iF].GetLength(jAgg, degrees[iF]) == N_rest * NoSpc_Agg); int i0Agg_Loc = mgMap.LocalUniqueIndex(iF, jAgg, N_rest * iSpc_Agg); int i0Agg = i0Agg_Loc + mgMap_Offset; Debug.Assert(i0Agg >= mgMap.Partitioning.i0); Debug.Assert(i0Agg < mgMap.Partitioning.iE); if (this.SpeciesIndexMapping[jAgg] == null) { Debug.Assert(NoSpc_Agg == 1); Debug.Assert(NoSpc_Agg == 1); // default branch: // use restriction/prolongation from un-cut basis // +++++++++++++++++++++++++++++++++++++++++++++ MultidimensionalArray Trf = base.CompositeBasis[jAgg]; for (int k = 0; k < K; k++) // loop over the cells which form the aggregated cell... { int jCell = AgCell[k]; int i0Full = mgMap.ProblemMapping.GlobalUniqueCoordinateIndex(iF, jCell, 0); var Block = Trf.ExtractSubArrayShallow(k, -1, -1); for (int nRow = 0; nRow < NROW; nRow++) { for (int nCol = 0; nCol < NCOL; nCol++) { RST[i0Agg + nRow, i0Full + nCol] = Block[nCol, nRow]; } } } } else { int NoSpc = this.NoOfSpecies[jAgg]; int[,] sim = SpeciesIndexMapping[jAgg]; Debug.Assert(sim.GetLength(0) == NoSpc); Debug.Assert(sim.GetLength(1) == K); MultidimensionalArray Trf; if (this.XCompositeBasis[jAgg] == null || this.XCompositeBasis[jAgg][iSpc_Agg] == null) { Trf = base.CompositeBasis[jAgg]; } else { Trf = this.XCompositeBasis[jAgg][iSpc_Agg]; } for (int k = 0; k < K; k++) // loop over the cells which form the aggregated cell... { int jCell = AgCell[k]; int iSpcBase = sim[iSpc_Agg, k]; if (iSpcBase < 0) { //for(int n = 0; n < N; n++) // FulCoords[k, n] = 0; } else { int i0Full = mgMap.ProblemMapping.GlobalUniqueCoordinateIndex(iF, jCell, iSpcBase * N_full); var Block = Trf.ExtractSubArrayShallow(k, -1, -1); for (int nRow = 0; nRow < NROW; nRow++) { for (int nCol = 0; nCol < NCOL; nCol++) { RST[i0Agg + nRow, i0Full + nCol] = Block[nCol, nRow]; } } } } } } } } //return RST; }
/// <summary> /// initiates the send/receive - processes and returns immediately; /// Every call of this method must be matched by a later call to <see cref="TransceiveFinish"/>; /// </summary> public void TransceiveStartImReturn() { ilPSP.MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD); var Para = m_master.iParallel; int[] sndProc = Para.ProcessesToReceiveFrom; // yes, intentional int MyRank; csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out MyRank); Array.Clear(this.rqst, 0, this.rqst.Length); unsafe { // Sending ... // ----------- // over all processes to which we have to send data to ... for (int i = 0; i < sndProc.Length; i++) { // destination processor and external cell range int pDest = sndProc[i]; int J0 = Para.RcvCommListsInsertIndex[pDest]; int JE = Para.RcvCommListsNoOfItems[pDest] + J0; // fill send buffer var SendBuffer = SendBuffers[i]; int extOffset = m_map.LocalLength; int cnt = 0; for (int jCell = J0; jCell < JE; jCell++) { Debug.Assert(jCell >= m_master.iLogicalCells.NoOfLocalUpdatedCells); int N = m_map.GetLength(jCell); int i0 = m_map.LocalUniqueIndex(0, jCell, 0); for (int n = 0; n < N; n++) { SendBuffer[cnt] = this.m_Vector_Ext[i0 + n - extOffset]; cnt++; } } Debug.Assert(cnt == SendBuffers[i].Length); // MPI send SendBufferPin[i] = GCHandle.Alloc(SendBuffers[i], GCHandleType.Pinned); csMPI.Raw.Issend(Marshal.UnsafeAddrOfPinnedArrayElement(SendBuffers[i], 0), SendBuffers[i].Length, csMPI.Raw._DATATYPE.DOUBLE, pDest, 4442 + MyRank, csMPI.Raw._COMM.WORLD, out rqst[i]); } } }
/// <summary> /// The core of the masking /// Generates index lists and the Ni0-struct-list corresponding to mask /// and is called by the child classes: local and external mask /// Note: the smallest unit are DG sub blocks! /// </summary> protected void GenerateAllMasks() { int NoOfCells = m_NoOfCells; int NoOfVariables = m_NoOfVariables; int[][] NoOfSpecies = m_NoOfSpecies; int[] DGdegreeP1 = m_DGdegree.CloneAs(); for (int iDG = 0; iDG < DGdegreeP1.Length; iDG++) { DGdegreeP1[iDG] += 1; } List <extNi0> ListNi0 = new List <extNi0>(); List <int> Globalint = new List <int>(); List <int> Localint = new List <int>(); List <int> SubBlockIdx = new List <int>(); int SubOffset = m_SubBlockOffset; // 0 for local mask and BMLoc.LocalDof for external mask int Ni0Length = 0; int MaskLen = 0; int prevLocie = m_map.LocalNoOfBlocks; var tmpCell = new List <extNi0[][][]>(); // local caching of filter functions // ensures that functions are not re-allocated during the loops var CellInstruction = m_sbs.CellFilter; var VarInstruction = m_sbs.VariableFilter; var SpecInstruction = m_sbs.SpeciesFilter; var ModeInstruction = m_sbs.ModeFilter; bool emptysel = true; // loop over cells... for (int iLoc = 0; iLoc < NoOfCells; iLoc++) { int jLoc = m_CellOffset + iLoc; //to address correctly, external cells offset has to be concidered, you know ... emptysel &= !CellInstruction(jLoc); //for testing if the entire selection is empty, which hopefully only can happen at the level of cells if (!CellInstruction(jLoc)) { continue; } var tmpVar = new List <extNi0[][]>(); // loop over variables... for (int iVar = 0; iVar < NoOfVariables; iVar++) { if (!VarInstruction(jLoc, iVar)) { continue; } var tmpSpc = new List <extNi0[]>(); // loop over species... for (int iSpc = 0; iSpc < NoOfSpecies[iLoc][iVar]; iSpc++) { if (!SpecInstruction(jLoc, iVar, iSpc)) { continue; } int GlobalOffset = m_map.GlobalUniqueIndex(iVar, jLoc, iSpc, 0); int LocalOffset = m_map.LocalUniqueIndex(iVar, jLoc, iSpc, 0); var tmpMod = new List <extNi0>(); // loop over polynomial degrees... for (int degree = 0; degree < DGdegreeP1[iVar]; degree++) { if (ModeInstruction(jLoc, iVar, iSpc, degree)) { int GlobalModeOffset = m_Ni0[degree].i0 + GlobalOffset; int LocalModeOffset = m_Ni0[degree].i0 + LocalOffset; int ModeLength = m_Ni0[degree].N; var newNi0 = new extNi0(LocalModeOffset, GlobalModeOffset, SubOffset, ModeLength); SubOffset += ModeLength; // Fill int lists for (int i = 0; i < newNi0.N; i++) { Globalint.Add(newNi0.Gi0 + i); Localint.Add(newNi0.Li0 + i); SubBlockIdx.Add(newNi0.Si0 + i); MaskLen++; } // Fill Ni0 Lists tmpMod.Add(newNi0); Ni0Length++; ListNi0.Add(newNi0); Debug.Assert(m_map.LocalUniqueIndex(iVar, jLoc, iSpc, GetNp(degree) - 1) == LocalModeOffset + ModeLength - 1); } } if (tmpMod.Count > 0) { tmpSpc.Add(tmpMod.ToArray()); } } if (tmpSpc.Count > 0) { tmpVar.Add(tmpSpc.ToArray()); } } if (tmpVar.Count > 0) { tmpCell.Add(tmpVar.ToArray()); } } var tmpStructNi0 = tmpCell.ToArray(); int NumOfNi0 = 0; #if DEBUG for (int iCell = 0; iCell < tmpStructNi0.Length; iCell++) { for (int iVar = 0; iVar < tmpStructNi0[iCell].Length; iVar++) { for (int iSpc = 0; iSpc < tmpStructNi0[iCell][iVar].Length; iSpc++) { NumOfNi0 += tmpStructNi0[iCell][iVar][iSpc].Length; } } } #endif // an empty selection is allowed, // e.g. consider a combination of empty external and non empty local mask if (!emptysel) { Debug.Assert(ListNi0.GroupBy(x => x.Li0).Any(g => g.Count() == 1)); Debug.Assert(ListNi0.GroupBy(x => x.Gi0).Any(g => g.Count() == 1)); Debug.Assert(ListNi0.Count() == NumOfNi0); Debug.Assert(MaskLen <= m_LocalLength); Debug.Assert(Localint.GroupBy(x => x).Any(g => g.Count() == 1)); Debug.Assert(Globalint.GroupBy(x => x).Any(g => g.Count() == 1)); Debug.Assert(Localint.Count() == MaskLen); Debug.Assert(SubBlockIdx.Count() == MaskLen); Debug.Assert(SubBlockIdx.GroupBy(x => x).Any(g => g.Count() == 1)); } m_GlobalMask = Globalint; m_LocalMask = Localint; m_StructuredNi0 = tmpStructNi0; m_MaskLen = MaskLen; m_Ni0Len = Ni0Length; m_SubBlockMask = SubBlockIdx; }