/// <summary> /// /// </summary> /// <param name="TargetMappingIndex"></param> /// <param name="outputPartitioning"> /// Partitioning of the new grid, resp the return array. /// </param> /// <returns> /// - 1st index: cell index in new grid, correlates with <paramref name="outputPartitioning"/>. /// - 2nd index: enumeration over cells (in the old grid) which are combined in the new grid. /// For cells with refinement, always one entry, for cells which are coarsened a greater number of entries. /// If null, the cell is not changed. /// - content: Subdivision leaf index, correlates with 2nd index of <see cref="KrefS_SubdivLeaves"/>, can be used as an input to <see cref="GetSubdivBasisTransform(int, int, int)"/>. /// </returns> public int[][] GetTargetMappingIndex(IPartitioning outputPartitioning) { using (new FuncTrace()) { Debug.Assert(DestGlobalId.Length == MappingIndex.Length); Debug.Assert(OldGlobalId.Length == MappingIndex.Length); int oldJ = DestGlobalId.Length; // Caching // ======= if (m_TargetMappingIndex != null) { // caching if (m_TargetMappingIndex.Length != outputPartitioning.LocalLength) { throw new ArgumentException("Length mismatch of output list and output partition."); } return(m_TargetMappingIndex); } // local evaluation, prepare communication // ======================================= m_TargetMappingIndex = new int[outputPartitioning.LocalLength][]; int j0Dest = outputPartitioning.i0; // keys: processors which should receive data from this processor Dictionary <int, GetTargetMapping_Helper> AllSendData = new Dictionary <int, GetTargetMapping_Helper>(); for (int j = 0; j < oldJ; j++) { int[] MappingIndex_j = MappingIndex[j]; if (MappingIndex_j != null) { Debug.Assert(TargetIdx[j].Length == MappingIndex_j.Length); int L = MappingIndex_j.Length; for (int l = 0; l < L; l++) { int jDest = TargetIdx[j][l]; int MapIdx = MappingIndex_j[l]; if (outputPartitioning.IsInLocalRange(jDest)) { int[] destCollection = m_TargetMappingIndex[jDest - j0Dest]; ArrayTools.AddToArray(MapIdx, ref destCollection); m_TargetMappingIndex[jDest - j0Dest] = destCollection; } else { int targProc = outputPartitioning.FindProcess(jDest); GetTargetMapping_Helper dataTargPrc; if (!AllSendData.TryGetValue(targProc, out dataTargPrc)) { dataTargPrc = new GetTargetMapping_Helper(); AllSendData.Add(targProc, dataTargPrc); } dataTargPrc.TargetIndices.Add(jDest); dataTargPrc.Items.Add(MapIdx); } } } else { Debug.Assert(TargetIdx[j].Length == 1); } } // communication // ============= var AllRcvData = SerialisationMessenger.ExchangeData(AllSendData, outputPartitioning.MPI_Comm); foreach (var kv in AllRcvData) { int rcvProc = kv.Key; j0Dest = outputPartitioning.GetI0Offest(rcvProc); var TIdxs = kv.Value.TargetIndices; var TVals = kv.Value.Items; Debug.Assert(TIdxs.Count == TVals.Count); int L = TIdxs.Count; for (int l = 0; l < L; l++) { int idx = TIdxs[l] - j0Dest; Debug.Assert(outputPartitioning.IsInLocalRange(idx)); int[] destCollection = m_TargetMappingIndex[idx]; ArrayTools.AddToArray(TVals[idx], ref destCollection); m_TargetMappingIndex[idx] = destCollection; } } // return // ====== return(m_TargetMappingIndex); } }
private static byte[] SyncEdgeTagsOverMPI(Dictionary <string, byte> EdgeTagNames_Reverse) { csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out int MyRank); var To0 = new Dictionary <int, KeyValuePair <string, byte>[]>(); if (MyRank > 0) { To0.Add(0, EdgeTagNames_Reverse.ToArray()); } var allData = SerialisationMessenger.ExchangeData(To0); bool[] usedEdgeTags = new bool[byte.MaxValue + 1]; foreach (var et in EdgeTagNames_Reverse.Values) { usedEdgeTags[et] = true; } byte GetNewEt() { for (int i = 1; i < usedEdgeTags.Length; i++) { if (i >= GridCommons.FIRST_PERIODIC_BC_TAG) { throw new ApplicationException("Running out of edge tags."); } if (usedEdgeTags[i] == false) { usedEdgeTags[i] = true; return((byte)i); } } throw new ApplicationException("Running out of edge tags."); } if (MyRank == 0) { foreach (var kv in allData) { var backData = kv.Value; for (int i = 0; i < backData.Length; i++) { if (EdgeTagNames_Reverse.ContainsKey(backData[i].Key)) { } else { byte sugKey = backData[i].Value; if (usedEdgeTags[sugKey]) { sugKey = GetNewEt(); } EdgeTagNames_Reverse.Add(backData[i].Key, sugKey); } } } } var AllEts = EdgeTagNames_Reverse.ToArray().MPIBroadcast(0); byte[] EtTanslations = (byte.MaxValue + 1).ForLoop(i => (byte)i); bool AnyTranslation = false; if (MyRank > 0) { foreach (var kv in AllEts) { if (EdgeTagNames_Reverse.ContainsKey(kv.Key)) { byte oldVal = EdgeTagNames_Reverse[kv.Key]; byte newVal = kv.Value; AnyTranslation = AnyTranslation | (oldVal != newVal); EdgeTagNames_Reverse[kv.Key] = newVal; EtTanslations[oldVal] = newVal; } } } AnyTranslation = AnyTranslation.MPIOr(); if (AnyTranslation) { return(EtTanslations); } else { return(null); } }
public void ApplyToVector <I>(IList <I> input, IList <I[]> output, IPartitioning outputPartitioning) { using (new FuncTrace()) { Debug.Assert(DestGlobalId.Length == MappingIndex.Length); Debug.Assert(OldGlobalId.Length == MappingIndex.Length); int oldJ = DestGlobalId.Length; if (input.Count != oldJ) { throw new ArgumentException("Mismatch between input vector length and current data length."); } if (output.Count != outputPartitioning.LocalLength) { throw new ArgumentException("Length mismatch of output list and output partition."); } int j0Dest = outputPartitioning.i0; // keys: processors which should receive data from this processor Dictionary <int, ApplyToVector_Helper <I> > AllSendData = new Dictionary <int, ApplyToVector_Helper <I> >(); for (int j = 0; j < oldJ; j++) { I data_j = input[j]; foreach (int jDest in TargetIdx[j]) { if (outputPartitioning.IsInLocalRange(jDest)) { I[] destCollection = output[jDest - j0Dest]; ArrayTools.AddToArray(data_j, ref destCollection); output[jDest - j0Dest] = destCollection; } else { int targProc = outputPartitioning.FindProcess(jDest); ApplyToVector_Helper <I> dataTargPrc; if (!AllSendData.TryGetValue(targProc, out dataTargPrc)) { dataTargPrc = new ApplyToVector_Helper <I>(); AllSendData.Add(targProc, dataTargPrc); } dataTargPrc.TargetIndices.Add(jDest); dataTargPrc.Items.Add(data_j); } } } var AllRcvData = SerialisationMessenger.ExchangeData(AllSendData, outputPartitioning.MPI_Comm); foreach (var kv in AllRcvData) { int rcvProc = kv.Key; j0Dest = outputPartitioning.GetI0Offest(rcvProc); var TIdxs = kv.Value.TargetIndices; var TVals = kv.Value.Items; Debug.Assert(TIdxs.Count == TVals.Count); int L = TIdxs.Count; for (int l = 0; l < L; l++) { int idx = TIdxs[l] - j0Dest; Debug.Assert(outputPartitioning.IsInLocalRange(idx)); I[] destCollection = output[idx]; ArrayTools.AddToArray(TVals[idx], ref destCollection); output[idx] = destCollection; } } } }
/// <summary> /// Like <see cref="Evaluate(double, IEnumerable{DGField}, MultidimensionalArray, double, MultidimensionalArray, BitArray, int[])"/>, /// but with MPI-Exchange /// </summary> public int EvaluateParallel(double alpha, IEnumerable <DGField> Flds, MultidimensionalArray Points, double beta, MultidimensionalArray Result, BitArray UnlocatedPoints = null) { using (new FuncTrace()) { MPICollectiveWatchDog.Watch(); int L = Points != null ? Points.NoOfRows : 0; int MPIsize = m_Context.MpiSize; int D = m_Context.SpatialDimension; if (UnlocatedPoints != null) { if (UnlocatedPoints.Length != L) { throw new ArgumentException("Length mismatch"); } } // evaluate locally // ================ var unlocated = new System.Collections.BitArray(L); int NoOfUnlocated; if (L > 0) { NoOfUnlocated = this.Evaluate(alpha, Flds, Points, beta, Result, unlocated); } else { NoOfUnlocated = 0; } // return, if there are no unlocalized points // ========================================== int TotNoOfUnlocated = NoOfUnlocated.MPISum(); if (TotNoOfUnlocated <= 0) { if (UnlocatedPoints != null) { UnlocatedPoints.SetAll(false); } return(0); } // copy unlocalized to separate array // ================================== double[,] localUnlocated = new double[NoOfUnlocated, D]; // MultidimensionalArray does not allow zero length -- so use double[,] instead int[] IndexToOrgIndex = new int[NoOfUnlocated]; int u = 0; for (int i = 0; i < L; i++) { if (unlocated[i]) { localUnlocated.SetRowPt(u, Points.GetRowPt(i)); IndexToOrgIndex[u] = i; u++; } } Debug.Assert(u == NoOfUnlocated); // collect on all ranks -- this won't scale well, but it may work // ============================================================== MultidimensionalArray globalUnlocated; int[] WhoIsInterestedIn; // index: point index, corresponds with 'globalUnlocated' rows; content: rank which needs the result int[] OriginalIndex; // index: detto; content: index which the point had on the processor that sent it. double[][,] __globalUnlocated; int LL; { __globalUnlocated = localUnlocated.MPIAllGatherO(); Debug.Assert(__globalUnlocated.Length == MPIsize); Debug.Assert(__globalUnlocated.Select(aa => aa.GetLength(0)).Sum() == TotNoOfUnlocated); Debug.Assert(__globalUnlocated[m_Context.MpiRank].GetLength(0) == NoOfUnlocated); LL = TotNoOfUnlocated - NoOfUnlocated; if (LL > 0) { globalUnlocated = MultidimensionalArray.Create(LL, D); } else { globalUnlocated = null; } WhoIsInterestedIn = new int[LL]; OriginalIndex = new int[LL]; int g = 0; for (int r = 0; r < MPIsize; r++) // concat all point arrays from all processors { if (r == m_Context.MpiRank) { continue; } double[,] __globalPart = __globalUnlocated[r]; int Lr = __globalPart.GetLength(0); if (Lr > 0) { globalUnlocated.ExtractSubArrayShallow(new[] { g, 0 }, new[] { g + Lr - 1, D - 1 }).Acc2DArray(1.0, __globalPart); } for (int i = 0; i < Lr; i++) { WhoIsInterestedIn[i + g] = r; OriginalIndex[i + g] = i; } g += Lr; } } // try to evaluate the so-far-unlocalized points // --------------------------------------------- var unlocated2 = new System.Collections.BitArray(LL); var Result2 = LL > 0 ? MultidimensionalArray.Create(LL, Flds.Count()) : null; int NoOfUnlocated2 = LL > 0 ? this.Evaluate(1.0, Flds, globalUnlocated, 0.0, Result2, unlocated2) : 0; // backward MPI sending // -------------------- IDictionary <int, EvaluateParallelHelper> resultFromOtherProcs; { var backSend = new Dictionary <int, EvaluateParallelHelper>(); for (int ll = 0; ll < LL; ll++) { if (!unlocated2[ll]) { int iTarget = WhoIsInterestedIn[ll]; Debug.Assert(iTarget != m_Context.MpiRank); if (!backSend.TryGetValue(iTarget, out EvaluateParallelHelper eph)) { eph = new EvaluateParallelHelper(); backSend.Add(iTarget, eph); } eph.OriginalIndices.Add(OriginalIndex[ll]); eph.Results.Add(Result2.GetRow(ll)); } } resultFromOtherProcs = SerialisationMessenger.ExchangeData(backSend); } // fill the results from other processors // ====================================== foreach (var res in resultFromOtherProcs.Values) { int K = res.OriginalIndices.Count(); Debug.Assert(res.OriginalIndices.Count == res.Results.Count); for (int k = 0; k < K; k++) { int iOrg = IndexToOrgIndex[res.OriginalIndices[k]]; if (unlocated[iOrg] == true) { NoOfUnlocated--; } unlocated[iOrg] = false; Result.AccRow(iOrg, alpha, res.Results[k]); } } // Return // ====== if (UnlocatedPoints != null) { for (int l = 0; l < L; l++) { UnlocatedPoints[l] = unlocated[l]; } } return(NoOfUnlocated); } }
/// <summary> /// Computes the neighbor cells globally (i.e. over all MPI processors) for each local cell. /// </summary> /// <returns> /// <param name="IncludeBcCells"> /// If true, also the boundary condition cells (<see cref="BcCells"/>) will be included in the output array. /// </param> /// Cell-wise neighborship information: /// - index: local cell index <em>j</em>, i.e. correlates with <see cref="Cells"/>; if <paramref name="IncludeBcCells"/> is true, /// the information for boundary cells is added after the information for cells. /// - content: for the index <em>j</em> the set of neighbor cells. If the global index (<see cref="Neighbour.Neighbour_GlobalIndex"/>) /// is greater or equal than the global number of cells (<see cref="NumberOfCells"/>) the neighbor is a boundary condition cell, /// (<see cref="BcCells"/>). /// </returns> public IEnumerable <Neighbour>[] GetCellNeighbourship(bool IncludeBcCells) { ilPSP.MPICollectiveWatchDog.Watch(); using (new FuncTrace()) { var ftNeigh = GetFaceTagsNeigbourIndices(IncludeBcCells); var NPart = this.NodePartitioning; int K = NPart.LocalLength; int k0 = NPart.i0; int J = this.NoOfUpdateCells; int J_BC = IncludeBcCells ? this.NoOfBcCells : 0; int j0 = this.CellPartitioning.i0; int Jglob = this.CellPartitioning.TotalLength; int j0Bc = this.BcCellPartitioning.i0; // Which cells make use of a particular node? //------------------------------------------- // Index: Node index // Entry: Enumeration of global indices of cells that use this // particular node List <int>[] Nodes2Cells = new List <int> [K]; { for (int k = 0; k < K; k++) { Nodes2Cells[k] = new List <int>(); } // key: MPI processor rank // value: information packet Dictionary <int, List <NodeCellIndexPair> > Y = new Dictionary <int, List <NodeCellIndexPair> >(); for (int j = 0; j < (J + J_BC); j++) { Element Cell_j; RefElement Kref; int jCell_glob; if (j < J) { Cell_j = this.Cells[j]; Kref = this.m_RefElements.Single(KK => KK.SupportedCellTypes.Contains(Cell_j.Type)); jCell_glob = j + j0; } else { Cell_j = this.BcCells[j - J]; Kref = this.m_EdgeRefElements.Single(KK => KK.SupportedCellTypes.Contains(Cell_j.Type)); jCell_glob = (j - J) + j0Bc + Jglob; } var CellNodes = Cell_j.NodeIndices; if (CellNodes.Length != Kref.NoOfVertices) { throw new ApplicationException("error in data structure."); } foreach (int NodeId in CellNodes) { int target_prozi = NPart.FindProcess(NodeId); if (target_prozi == MyRank) { Nodes2Cells[NodeId - k0].Add(jCell_glob); } else { NodeCellIndexPair Packet; Packet.NodeId = NodeId; Packet.GlobalCellIndex = jCell_glob; List <NodeCellIndexPair> Z; if (!Y.TryGetValue(target_prozi, out Z)) { Z = new List <NodeCellIndexPair>(); Y.Add(target_prozi, Z); } Z.Add(Packet); } } } var W = SerialisationMessenger.ExchangeData(Y, csMPI.Raw._COMM.WORLD); foreach (var wp in W.Values) { foreach (NodeCellIndexPair Packet in wp) { Nodes2Cells[Packet.NodeId - k0].Add(Packet.GlobalCellIndex); } } } // For every cell, for every vertex in this cell: // Which other cells die also use this node? //----------------------------------------------- // 1st index: Local cell index // 2nd index: Cell vertex index // 3rd index: Collection of 'peer' cells int[][][] NodePeers = new int[J + J_BC][][]; { for (int j = 0; j < J + J_BC; j++) { Element Cell_j; if (j < J) { Cell_j = this.Cells[j]; } else { Cell_j = this.BcCells[j - J]; } NodePeers[j] = new int[Cell_j.NodeIndices.Length][]; } Dictionary <int, List <NodeCellListPair> > Y = new Dictionary <int, List <NodeCellListPair> >(); var CPart = this.CellPartitioning; var BcPart = this.BcCellPartitioning; for (int k = 0; k < K; k++) // loop over locally assigned nodes { int k_node = k + k0; var cell_list = Nodes2Cells[k].ToArray(); foreach (int jCell in cell_list) // loop over all cells that use node 'k' { int cell_proc; int local_offset; if (jCell < Jglob) { // normal cell cell_proc = CPart.FindProcess(jCell); local_offset = j0; } else { // boundary condition cell cell_proc = BcPart.FindProcess(jCell - Jglob); local_offset = Jglob + j0Bc; } if (cell_proc == MyRank) { int jCell_loc = jCell - local_offset; int kC; bool bfound = false; Element Cell_j; int oo; if (jCell < Jglob) { // normal cell Cell_j = this.Cells[jCell_loc]; oo = 0; } else { // boundary condition cell Cell_j = this.BcCells[jCell_loc]; oo = J; } for (kC = 0; kC < Cell_j.NodeIndices.Length; kC++) { if (Cell_j.NodeIndices[kC] == k_node) { bfound = true; break; } } if (!bfound) { throw new ApplicationException("error in algorithm."); } NodePeers[jCell_loc + oo][kC] = cell_list; } else { NodeCellListPair A; A.NodeId = k_node; A.CellList = cell_list; List <NodeCellListPair> Z; if (!Y.TryGetValue(cell_proc, out Z)) { Z = new List <NodeCellListPair>(); Y.Add(cell_proc, Z); } Z.Add(A); } } } var W = SerialisationMessenger.ExchangeData(Y, csMPI.Raw._COMM.WORLD); foreach (var wp in W.Values) { foreach (var P in wp) { int k_node = P.NodeId; int[] cell_list = P.CellList; foreach (int jCell in cell_list) { int cell_proc; int local_offset; if (jCell < Jglob) { // normal cell cell_proc = CPart.FindProcess(jCell); local_offset = j0; } else { // boundary condition cell cell_proc = BcPart.FindProcess(jCell - Jglob); local_offset = Jglob + j0Bc; } if (cell_proc == MyRank) { int jCell_loc = jCell - local_offset; Element Cell_j; int oo; if (jCell < Jglob) { // normal cell Cell_j = this.Cells[jCell_loc]; oo = 0; } else { // boundary condition cell Cell_j = this.BcCells[jCell_loc]; oo = J; } int kC; bool bfound = false; for (kC = 0; kC < Cell_j.NodeIndices.Length; kC++) { if (Cell_j.NodeIndices[kC] == k_node) { bfound = true; break; } } if (!bfound) { throw new ApplicationException("error in algorithm."); } NodePeers[jCell_loc + oo][kC] = cell_list; } } } } } // Assemble final result // --------------------- IEnumerable <Neighbour>[] CellNeighbours; { CellNeighbours = new IEnumerable <Neighbour> [J + J_BC]; for (int j = 0; j < J + J_BC; j++) // loop over cells //var Cell_j = this.Cells[j]; //int jCellGlob = j + j0; //var Kref = this.m_GridSimplices.Single(KK => KK.SupportedTypes.Contains(Cell_j.Type)); { Element Cell_j; RefElement Kref; int jCellGlob; if (j < J) { Cell_j = this.Cells[j]; Kref = this.m_RefElements.Single(KK => KK.SupportedCellTypes.Contains(Cell_j.Type)); jCellGlob = j + j0; } else { Cell_j = this.BcCells[j - J]; Kref = this.m_EdgeRefElements.Single(KK => KK.SupportedCellTypes.Contains(Cell_j.Type)); jCellGlob = (j - J) + j0Bc + Jglob; } var Cell_j_Neighs = new List <Neighbour>(); CellNeighbours[j] = Cell_j_Neighs; // find neighbor cells connected via grid nodes // -------------------------------------------- if (j < J) { //normal cells: match faces var faceVtx = Kref.FaceToVertexIndices; int[][] B = new int[faceVtx.GetLength(1)][]; for (int _iface = 0; _iface < Kref.NoOfFaces; _iface++) // loop over faces of cell 'j' (local index) resp. 'jCellGlob' (global index) { for (int iv = 0; iv < B.Length; iv++) { B[iv] = NodePeers[j][faceVtx[_iface, iv]]; } int NeighIdx = Intersect(B, jCellGlob); if (NeighIdx >= 0) { Neighbour nCN = default(Neighbour); nCN.Neighbour_GlobalIndex = NeighIdx; nCN.CellFaceTag.FaceIndex = _iface; nCN.CellFaceTag.ConformalNeighborship = true; Cell_j_Neighs.Add(nCN); } } } else { // boundary-condition cell: match the whole element int[][] B = new int[Kref.NoOfVertices][]; for (int iv = 0; iv < B.Length; iv++) { B[iv] = NodePeers[j][iv]; } int NeighIdx = Intersect(B, jCellGlob); if (NeighIdx >= 0) { Neighbour nCN = default(Neighbour); nCN.Neighbour_GlobalIndex = NeighIdx; nCN.CellFaceTag.FaceIndex = -1; nCN.CellFaceTag.ConformalNeighborship = true; Cell_j_Neighs.Add(nCN); } } // find neighbor cells connected via CellFaceTag's // ----------------------------------------------- var otherNeighbours = ftNeigh[j]; // ftNeigh is the result of CellFaceTag-based connectivity if (j < J) { var _Cell_j = (Cell)Cell_j; Debug.Assert(((otherNeighbours == null ? 0 : otherNeighbours.Length) == ((_Cell_j.CellFaceTags == null) ? 0 : _Cell_j.CellFaceTags.Length))); if (otherNeighbours != null) { for (int w = 0; w < otherNeighbours.Length; w++) { Debug.Assert(_Cell_j.CellFaceTags[w].NeighCell_GlobalID < 0 == otherNeighbours[w] < 0); if (_Cell_j.CellFaceTags[w].NeighCell_GlobalID >= 0) { if (Cell_j_Neighs.Where(neigh => neigh.Neighbour_GlobalIndex == otherNeighbours[w]).Count() <= 0) // filter duplicates { Cell_j_Neighs.Add(new Neighbour() { Neighbour_GlobalIndex = otherNeighbours[w], CellFaceTag = _Cell_j.CellFaceTags[w], }); } } } } } else { var BcCell_j = (BCElement)Cell_j; Debug.Assert(((otherNeighbours == null ? 0 : otherNeighbours.Length) == ((BcCell_j.NeighCell_GlobalIDs == null) ? 0 : BcCell_j.NeighCell_GlobalIDs.Length))); if (otherNeighbours != null) { for (int w = 0; w < otherNeighbours.Length; w++) { Cell_j_Neighs.Add(new Neighbour() { Neighbour_GlobalIndex = otherNeighbours[w], CellFaceTag = new CellFaceTag() { EdgeTag = BcCell_j.EdgeTag, FaceIndex = int.MinValue, NeighCell_GlobalID = BcCell_j.NeighCell_GlobalIDs[w], ConformalNeighborship = BcCell_j.Conformal } }); } } } } } return(CellNeighbours); } }
/// <summary> /// ctor. /// </summary> /// <param name="M"></param> /// <param name="ExtCol"> /// key: processor rank 'p' <br/> /// value: a list of column indices (within the local range of columns of <paramref name="M"/>), /// which should be editable at rank 'p'. /// </param> public MsrExtMatrix(IMutableMatrixEx M, IDictionary <int, int[]> ExtCol) { this.ColPart = M.ColPartition; int i0Row = (int)M.RowPartitioning.i0, I = M.RowPartitioning.LocalLength, i0Col = (int)this.ColPart.i0, J = this.ColPart.LocalLength; Mtx = M; // init // ==== ColToRowLocal = new List <int> [ColPart.LocalLength]; for (int j = 0; j < ColToRowLocal.Length; j++) { ColToRowLocal[j] = new List <int>(); } // build Column to row - mapping // ============================= ColToRowExternal = new Dictionary <int, List <int> >(); // key: global column index j, within the range of processor 'p' // values: global row indices SortedDictionary <int, List <int> > ColForProc = new SortedDictionary <int, List <int> >(); // key: MPI processor index 'p' // values: a set of global column indices, within the range of processor 'p', // that contain nonzero entries on this processor int[] col = null; int L; // loop over all rows... for (int i = 0; i < I; i++) { L = M.GetOccupiedColumnIndices(i + i0Row, ref col); // loop over all nonzero entries in the row... for (int l = 0; l < L; l++) { int ColIndex = col[l]; int localColInd = ColIndex - i0Col; if (localColInd >= 0 && localColInd < J) { // column of 'entry' is within the local range of this processor // + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ColToRowLocal[localColInd].Add(i + i0Row); } else { // column of 'entry' belongs to external processor 'proc' // + + + + + + + + + + + + + + + + + + + + + + + + + + + + int proc = this.ColPart.FindProcess(ColIndex); { //SortedDictionary<int, List<int>> ColToRowExt_proc; //if (!ColToRowExternal.ContainsKey(proc)) { // ColToRowExt_proc = new SortedDictionary<int, List<int>>(); // ColToRowExternal.Add(proc, ColToRowExt_proc); //} else { // ColToRowExt_proc = ColToRowExternal[proc]; //} int j = ColIndex; List <int> Rows4Col; if (!ColToRowExternal.ContainsKey(j)) { Rows4Col = new List <int>(); ColToRowExternal.Add(j, Rows4Col); } else { Rows4Col = ColToRowExternal[j]; } Rows4Col.Add(i + i0Row); } { List <int> ColForProc_proc; if (!ColForProc.ContainsKey(proc)) { ColForProc_proc = new List <int>(); ColForProc.Add(proc, ColForProc_proc); } else { ColForProc_proc = ColForProc[proc]; } if (!ColForProc_proc.Contains(ColIndex)) { ColForProc_proc.Add(ColIndex); } } } } } // communicate // =========== //SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw.MPI_COMM_WORLD); //{ // foreach (int proc in ColToRowExternal.Keys) { // sms.SetCommPath(proc); // } // sms.CommitCommPaths(); // // send // foreach (int proc in ColToRowExternal.Keys) { // SortedDictionary<int, List<int>> ColToRowExt_proc = ColToRowExternal[proc]; // SortedList _ColToRowExt_proc = new SortedList(); // foreach (int iCol in ColToRowExt_proc.Keys) // _ColToRowExt_proc.Add(iCol, ColToRowExt_proc[iCol].ToArray()); // } // // receive // int p; SortedList rcv; // sms.GetNext(out p, out rcv); // while (rcv != null) { // foreach (int col in rcv.Keys) { // int[] rowList = (int[])rcv[col]; // ColToRowLocal[col].AddRange(rowList); // } // sms.GetNext(out p, out rcv); // } //} //sms.Dispose(); // build 'ColProcessors' // ===================== //ColProcessors = new List<int>[ColToRowLocal.Length]; //for (int j = 0; j < ColToRowLocal.Length; j++) { // List<int> mpiRank = null; // foreach (int rowind in ColToRowLocal[j]) { // int riloc = rowind - i0Row; // if (riloc < 0 || riloc >= I) { // if (mpiRank == null) // mpiRank = new List<int>(); // mpiRank.Add(M.RowPartiton.FindProcess(rowind)); // } // ColProcessors[j] = mpiRank; // } //} // communicate: build 'ColProcessors' // ================================== { ColProcessors = new List <int> [ColPart.LocalLength]; SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD); sms.SetCommPathsAndCommit(ColForProc.Keys); foreach (int proc in ColForProc.Keys) { sms.Transmit(proc, ColForProc[proc].ToArray()); } int i0Loc = (int)ColPart.i0; int rcvproc; int[] ColIndices; while (sms.GetNext(out rcvproc, out ColIndices)) { int Rank; { csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out Rank); //Console.WriteLine("P# " + Rank + ": receiving from P# " + rcvproc); } foreach (int ColInd in ColIndices) { int localColInd = ColInd - i0Loc; if (localColInd < 0 || localColInd >= ColPart.LocalLength) { throw new IndexOutOfRangeException("internal error"); } if (ColProcessors[localColInd] == null) { ColProcessors[localColInd] = new List <int>(); } if (ColProcessors[localColInd].Contains(rcvproc)) { throw new ApplicationException("internal error."); } ColProcessors[localColInd].Add(rcvproc); } } sms.Dispose(); } if (ExtCol != null) { var send = new Dictionary <int, List <Tuple <int, List <int> > > >(); int myRank = M.RowPartitioning.MpiRank; foreach (var kv in ExtCol) { int rank = kv.Key; int[] ColIdx = kv.Value; var sendToRank = new List <Tuple <int, List <int> > >(); foreach (int iCol in ColIdx) { List <int> c2p = ColProcessors[iCol - i0Col]; var t = new Tuple <int, List <int> >(iCol, c2p != null ? new List <int>(c2p) : new List <int>()); t.Item2.Add(myRank); sendToRank.Add(t); } send.Add(rank, sendToRank); } var receive = SerialisationMessenger.ExchangeData(send, csMPI.Raw._COMM.WORLD); ColProcessorsExternal = new Dictionary <int, List <int> >(); foreach (var kv in receive) { var val = kv.Value; foreach (var t in val) { int iCol = t.Item1; List <int> ranks = t.Item2; int iMyRank = ranks.IndexOf(myRank); if (iMyRank >= 0) { ranks.RemoveAt(iMyRank); } ColProcessorsExternal.Add(t.Item1, t.Item2); Debug.Assert(this.ColPart.FindProcess(t.Item1) == kv.Key); } } #if DEBUG foreach (var procList in ColProcessorsExternal.Values) { Debug.Assert(procList.Contains(myRank) == false); } #endif } }
/// <summary> /// Resorts a vector according to this permutation, i.e. the /// <em>j</em>-th item of the input vector is copied to the /// <see cref="Values"/>[j]-th entry of the output vector. /// </summary> /// <param name="input"> /// Input vector, length must be equal to the length of this permutation, unchanged on exit. /// </param> /// <param name="output"> /// On exit, <paramref name="output"/>[<see cref="Values"/>[j]] = <paramref name="input"/>[j] /// </param> public void ApplyToVector <I>(IList <I> input, IList <I> output, IPartitioning outputPartitioning) { using (new FuncTrace()) { if (input.Count != this.LocalLength) { throw new ArgumentException("wrong size of input vector."); } if (output.Count != outputPartitioning.LocalLength) { throw new ArgumentException("wrong size of output vector."); } long[] TargetInd = this.Values; // keys: processors which should receive data from this processor Dictionary <int, ApplyToVector_Helper <I> > sendData = new Dictionary <int, ApplyToVector_Helper <I> >(); int out_myI0 = outputPartitioning.i0; int out_nextI0 = out_myI0 + outputPartitioning.LocalLength; int J = this.Partitioning.LocalLength; for (int j = 0; j < J; j++) { if (out_myI0 <= TargetInd[j] && TargetInd[j] < out_nextI0) { // target index located on this processor output[(int)(TargetInd[j] - out_myI0)] = input[j]; } else { // target index located on other processor int TargProc = outputPartitioning.FindProcess(TargetInd[j]); ApplyToVector_Helper <I> sendData_TargProc = null; if (!sendData.TryGetValue(TargProc, out sendData_TargProc)) { sendData_TargProc = new ApplyToVector_Helper <I>(); sendData.Add(TargProc, sendData_TargProc); } sendData_TargProc.Items.Add(input[j]); sendData_TargProc.TargetIndices.Add(TargetInd[j]); } } var rcvData = SerialisationMessenger.ExchangeData( sendData, MPI.Wrappers.csMPI.Raw._COMM.WORLD); foreach (var rcvPkt in rcvData.Values) { int K = rcvPkt.Items.Count; Debug.Assert(rcvPkt.Items.Count == rcvPkt.TargetIndices.Count); for (int k = 0; k < K; k++) { int locIdx = (int)(rcvPkt.TargetIndices[k]) - out_myI0; output[locIdx] = rcvPkt.Items[k]; } } } }