/// <summary> /// MPI update of a bit-array /// </summary> /// <param name="b"></param> /// <param name="GridData"></param> static public void MPIExchange(this BitArray b, IGridData GridData) { if (b.Length != GridData.iLogicalCells.NoOfCells) { throw new ArgumentException("length must be equal to number of cells.", "b"); } if (GridData.CellPartitioning.MpiSize > 1) { // external { int rank, size; csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank); csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out size); // setup messenger SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD); sms.SetCommPathsAndCommit(GridData.iParallel.ProcessesToSendTo); // send data for (int p = 0; p < size; p++) { int[] sendlist = GridData.iParallel.SendCommLists[p]; if (sendlist == null) { continue; } int L = sendlist.Length; System.Collections.BitArray packet_for_p = new System.Collections.BitArray(L, false); for (int l = 0; l < L; l++) { packet_for_p[l] = b[sendlist[l]]; } sms.Transmitt(p, packet_for_p); } // receive data System.Collections.BitArray rcv_dat; int rcv_rank; while (sms.GetNext(out rcv_rank, out rcv_dat)) { int insertAt = GridData.iParallel.RcvCommListsInsertIndex[rcv_rank]; if (GridData.iParallel.RcvCommListsNoOfItems[rcv_rank] != rcv_dat.Count) { throw new ApplicationException("internal error."); } int C = rcv_dat.Count; for (int i = 0; i < C; i++) { b[insertAt + i] = rcv_dat[i]; } } // dispose sms.Dispose(); } } }
/// <summary> /// ctor. /// </summary> /// <param name="M"></param> /// <param name="ExtCol"> /// key: processor rank 'p' <br/> /// value: a list of column indices (within the local range of columns of <paramref name="M"/>), /// which should be editable at rank 'p'. /// </param> public MsrExtMatrix(IMutableMatrixEx M, IDictionary <int, int[]> ExtCol) { this.ColPart = M.ColPartition; int i0Row = (int)M.RowPartitioning.i0, I = M.RowPartitioning.LocalLength, i0Col = (int)this.ColPart.i0, J = this.ColPart.LocalLength; Mtx = M; // init // ==== ColToRowLocal = new List <int> [ColPart.LocalLength]; for (int j = 0; j < ColToRowLocal.Length; j++) { ColToRowLocal[j] = new List <int>(); } // build Column to row - mapping // ============================= ColToRowExternal = new Dictionary <int, List <int> >(); // key: global column index j, within the range of processor 'p' // values: global row indices SortedDictionary <int, List <int> > ColForProc = new SortedDictionary <int, List <int> >(); // key: MPI processor index 'p' // values: a set of global column indices, within the range of processor 'p', // that contain nonzero entries on this processor int[] col = null; int L; // loop over all rows... for (int i = 0; i < I; i++) { L = M.GetOccupiedColumnIndices(i + i0Row, ref col); // loop over all nonzero entries in the row... for (int l = 0; l < L; l++) { int ColIndex = col[l]; int localColInd = ColIndex - i0Col; if (localColInd >= 0 && localColInd < J) { // column of 'entry' is within the local range of this processor // + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ColToRowLocal[localColInd].Add(i + i0Row); } else { // column of 'entry' belongs to external processor 'proc' // + + + + + + + + + + + + + + + + + + + + + + + + + + + + int proc = this.ColPart.FindProcess(ColIndex); { //SortedDictionary<int, List<int>> ColToRowExt_proc; //if (!ColToRowExternal.ContainsKey(proc)) { // ColToRowExt_proc = new SortedDictionary<int, List<int>>(); // ColToRowExternal.Add(proc, ColToRowExt_proc); //} else { // ColToRowExt_proc = ColToRowExternal[proc]; //} int j = ColIndex; List <int> Rows4Col; if (!ColToRowExternal.ContainsKey(j)) { Rows4Col = new List <int>(); ColToRowExternal.Add(j, Rows4Col); } else { Rows4Col = ColToRowExternal[j]; } Rows4Col.Add(i + i0Row); } { List <int> ColForProc_proc; if (!ColForProc.ContainsKey(proc)) { ColForProc_proc = new List <int>(); ColForProc.Add(proc, ColForProc_proc); } else { ColForProc_proc = ColForProc[proc]; } if (!ColForProc_proc.Contains(ColIndex)) { ColForProc_proc.Add(ColIndex); } } } } } // communicate // =========== //SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw.MPI_COMM_WORLD); //{ // foreach (int proc in ColToRowExternal.Keys) { // sms.SetCommPath(proc); // } // sms.CommitCommPaths(); // // send // foreach (int proc in ColToRowExternal.Keys) { // SortedDictionary<int, List<int>> ColToRowExt_proc = ColToRowExternal[proc]; // SortedList _ColToRowExt_proc = new SortedList(); // foreach (int iCol in ColToRowExt_proc.Keys) // _ColToRowExt_proc.Add(iCol, ColToRowExt_proc[iCol].ToArray()); // } // // receive // int p; SortedList rcv; // sms.GetNext(out p, out rcv); // while (rcv != null) { // foreach (int col in rcv.Keys) { // int[] rowList = (int[])rcv[col]; // ColToRowLocal[col].AddRange(rowList); // } // sms.GetNext(out p, out rcv); // } //} //sms.Dispose(); // build 'ColProcessors' // ===================== //ColProcessors = new List<int>[ColToRowLocal.Length]; //for (int j = 0; j < ColToRowLocal.Length; j++) { // List<int> mpiRank = null; // foreach (int rowind in ColToRowLocal[j]) { // int riloc = rowind - i0Row; // if (riloc < 0 || riloc >= I) { // if (mpiRank == null) // mpiRank = new List<int>(); // mpiRank.Add(M.RowPartiton.FindProcess(rowind)); // } // ColProcessors[j] = mpiRank; // } //} // communicate: build 'ColProcessors' // ================================== { ColProcessors = new List <int> [ColPart.LocalLength]; SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD); sms.SetCommPathsAndCommit(ColForProc.Keys); foreach (int proc in ColForProc.Keys) { sms.Transmit(proc, ColForProc[proc].ToArray()); } int i0Loc = (int)ColPart.i0; int rcvproc; int[] ColIndices; while (sms.GetNext(out rcvproc, out ColIndices)) { int Rank; { csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out Rank); //Console.WriteLine("P# " + Rank + ": receiving from P# " + rcvproc); } foreach (int ColInd in ColIndices) { int localColInd = ColInd - i0Loc; if (localColInd < 0 || localColInd >= ColPart.LocalLength) { throw new IndexOutOfRangeException("internal error"); } if (ColProcessors[localColInd] == null) { ColProcessors[localColInd] = new List <int>(); } if (ColProcessors[localColInd].Contains(rcvproc)) { throw new ApplicationException("internal error."); } ColProcessors[localColInd].Add(rcvproc); } } sms.Dispose(); } if (ExtCol != null) { var send = new Dictionary <int, List <Tuple <int, List <int> > > >(); int myRank = M.RowPartitioning.MpiRank; foreach (var kv in ExtCol) { int rank = kv.Key; int[] ColIdx = kv.Value; var sendToRank = new List <Tuple <int, List <int> > >(); foreach (int iCol in ColIdx) { List <int> c2p = ColProcessors[iCol - i0Col]; var t = new Tuple <int, List <int> >(iCol, c2p != null ? new List <int>(c2p) : new List <int>()); t.Item2.Add(myRank); sendToRank.Add(t); } send.Add(rank, sendToRank); } var receive = SerialisationMessenger.ExchangeData(send, csMPI.Raw._COMM.WORLD); ColProcessorsExternal = new Dictionary <int, List <int> >(); foreach (var kv in receive) { var val = kv.Value; foreach (var t in val) { int iCol = t.Item1; List <int> ranks = t.Item2; int iMyRank = ranks.IndexOf(myRank); if (iMyRank >= 0) { ranks.RemoveAt(iMyRank); } ColProcessorsExternal.Add(t.Item1, t.Item2); Debug.Assert(this.ColPart.FindProcess(t.Item1) == kv.Key); } } #if DEBUG foreach (var procList in ColProcessorsExternal.Values) { Debug.Assert(procList.Contains(myRank) == false); } #endif } }