/// <summary> /// Set reference point for pressure by manipulating <paramref name="Matrix"/>, i.e. /// mean value for pressure correction (i.e. zeroth order polynomial coefficient) /// is set to zero in cell with GlobalID of zero, which is given by <paramref name="IndexRefPtPressure"/>. /// </summary> /// <param name="Matrix"></param> /// <param name="IndexRefPtPressure"></param> public static void SetRefPtPressure_Matrix(IMutableMatrixEx Matrix, int IndexRefPtPressure) { int i0 = Matrix.RowPartitioning.i0; int LocalLength = Matrix.RowPartitioning.LocalLength; if ((IndexRefPtPressure >= i0) && (IndexRefPtPressure < i0 + LocalLength)) { Matrix[IndexRefPtPressure, IndexRefPtPressure] = 1.0; int[] colIdx = Matrix.GetOccupiedColumnIndices(IndexRefPtPressure); foreach (int col in colIdx) { if (col != IndexRefPtPressure) { Matrix[IndexRefPtPressure, col] = 0.0; } } for (int row = 0; row < LocalLength; row++) { if ((row + i0) != IndexRefPtPressure) { Matrix[row + i0, IndexRefPtPressure] = 0.0; } } } else { for (int row = 0; row < LocalLength; row++) { Matrix[row + i0, IndexRefPtPressure] = 0.0; } } }
/// <summary> /// Returns a collection of all occupied columns in a the row <paramref name="RowIndex"/>; /// </summary> /// <returns> /// The column indices of non-zero/allocated entries. /// </returns> static public int[] GetOccupiedColumnIndices(this IMutableMatrixEx M, int iRow) { int[] idx = null; int L = M.GetOccupiedColumnIndices(iRow, ref idx); if (L != idx.Length) { Array.Resize(ref idx, L); } return(idx); }
/// <summary> /// matrix assembly; must be called by each implementation, /// </summary> /// <param name="M"></param> protected void PackMatrix(IMutableMatrixEx M) { ilPSP.MPICollectiveWatchDog.Watch(); IPartitioning rp = M.RowPartitioning; IPartitioning cp = m_ColPart; // define Comm List // ================ SortedDictionary <int, List <int> > CommLists = new SortedDictionary <int, List <int> >(); // keys: processor rank p // values: List of global indices, which processor p needs to send to this processor int Lr; double[] val = null; int[] col = null; int L = rp.LocalLength; int i0 = (int)rp.i0; for (int iLoc = 0; iLoc < L; iLoc++) // loop over all matrix rows... { int iGlob = i0 + iLoc; //MsrMatrix.MatrixEntry[] row = (asMsr==null) ? M.GetRow(iGlob) : asMsr.GetRowShallow(iGlob); Lr = M.GetOccupiedColumnIndices(iGlob, ref col); for (int j = 0; j < Lr; j++) // loop over all nonzero entries in row 'iGlob' { int jGlob = col[j]; if (cp.i0 <= jGlob && jGlob < (cp.i0 + cp.LocalLength)) { // Entry on current processor } else { int proc = cp.FindProcess(jGlob); // Entry on Processor proc if (!CommLists.ContainsKey(proc)) { CommLists.Add(proc, new List <int>()); } List <int> CommList_proc = CommLists[proc]; if (!CommList_proc.Contains(jGlob)) // a lot of room for optimization { CommList_proc.Add(jGlob); } } } } // sort com list // ============= { foreach (List <int> cl in CommLists.Values) { cl.Sort(); } } // define matrix // ============= { TempCSR intTmp = new TempCSR(); SortedDictionary <int, ExternalTmp> extTmp = new SortedDictionary <int, ExternalTmp>(); foreach (int proc in CommLists.Keys) { extTmp.Add(proc, new ExternalTmp()); } for (int iLoc = 0; iLoc < L; iLoc++) { int iGlob = i0 + iLoc; Lr = M.GetRow(iGlob, ref col, ref val); for (int j = 0; j < Lr; j++) { int jGlob = col[j]; double Value = val[j]; bool bIsDiag = (iGlob == jGlob); if (cp.i0 <= jGlob && jGlob < (cp.i0 + cp.LocalLength)) { // Entry on current processor intTmp.AddEntry(jGlob - (int)cp.i0, Value, bIsDiag); } else { int proc = cp.FindProcess(jGlob); // Entry on Processor proc List <int> CommList_proc = CommLists[proc]; int jloc = CommList_proc.IndexOf(jGlob); ExternalTmp et = extTmp[proc]; et.AddEntry(jloc, jGlob, Value); } } intTmp.NextRow(); foreach (ExternalTmp et in extTmp.Values) { et.NextRow(); } } m_LocalMtx = AssembleFinalFormat(intTmp); ExtMatrix = new Dictionary <int, External>(); foreach (int proc in extTmp.Keys) { ExtMatrix.Add(proc, extTmp[proc].GetFinalObj()); } } // send/receive & transform Comm lists // ==================================== { SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD); SortedDictionary <int, int[]> CommListsTo = new SortedDictionary <int, int[]>(); foreach (int proc in CommLists.Keys) { sms.SetCommPath(proc); } sms.CommitCommPaths(); foreach (int proc in CommLists.Keys) { sms.Transmitt(proc, CommLists[proc].ToArray()); } int _proc; int[] CommListReceived; sms.GetNext(out _proc, out CommListReceived); int Lcol = m_ColPart.LocalLength; int i0col = (int)m_ColPart.i0; while (CommListReceived != null) { // convert indices to local coordinates for (int i = 0; i < CommListReceived.Length; i++) { CommListReceived[i] -= i0col; // check: if (CommListReceived[i] < 0 || CommListReceived[i] >= Lcol) { throw new ApplicationException("internal error: something wrong with received Comm List."); } } CommListsTo.Add(_proc, CommListReceived); sms.GetNext(out _proc, out CommListReceived); } sms.Dispose(); m_SpmvCommPattern = new SpmvCommPattern(); m_SpmvCommPattern.ComLists = CommListsTo; } // record the number of elements which we receive // ============================================== { m_SpmvCommPattern.NoOfReceivedEntries = new Dictionary <int, int>(); foreach (int p in CommLists.Keys) { m_SpmvCommPattern.NoOfReceivedEntries.Add(p, CommLists[p].Count); } } }
/// <summary> /// ctor. /// </summary> /// <param name="M"></param> /// <param name="ExtCol"> /// key: processor rank 'p' <br/> /// value: a list of column indices (within the local range of columns of <paramref name="M"/>), /// which should be editable at rank 'p'. /// </param> public MsrExtMatrix(IMutableMatrixEx M, IDictionary <int, int[]> ExtCol) { this.ColPart = M.ColPartition; int i0Row = (int)M.RowPartitioning.i0, I = M.RowPartitioning.LocalLength, i0Col = (int)this.ColPart.i0, J = this.ColPart.LocalLength; Mtx = M; // init // ==== ColToRowLocal = new List <int> [ColPart.LocalLength]; for (int j = 0; j < ColToRowLocal.Length; j++) { ColToRowLocal[j] = new List <int>(); } // build Column to row - mapping // ============================= ColToRowExternal = new Dictionary <int, List <int> >(); // key: global column index j, within the range of processor 'p' // values: global row indices SortedDictionary <int, List <int> > ColForProc = new SortedDictionary <int, List <int> >(); // key: MPI processor index 'p' // values: a set of global column indices, within the range of processor 'p', // that contain nonzero entries on this processor int[] col = null; int L; // loop over all rows... for (int i = 0; i < I; i++) { L = M.GetOccupiedColumnIndices(i + i0Row, ref col); // loop over all nonzero entries in the row... for (int l = 0; l < L; l++) { int ColIndex = col[l]; int localColInd = ColIndex - i0Col; if (localColInd >= 0 && localColInd < J) { // column of 'entry' is within the local range of this processor // + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ColToRowLocal[localColInd].Add(i + i0Row); } else { // column of 'entry' belongs to external processor 'proc' // + + + + + + + + + + + + + + + + + + + + + + + + + + + + int proc = this.ColPart.FindProcess(ColIndex); { //SortedDictionary<int, List<int>> ColToRowExt_proc; //if (!ColToRowExternal.ContainsKey(proc)) { // ColToRowExt_proc = new SortedDictionary<int, List<int>>(); // ColToRowExternal.Add(proc, ColToRowExt_proc); //} else { // ColToRowExt_proc = ColToRowExternal[proc]; //} int j = ColIndex; List <int> Rows4Col; if (!ColToRowExternal.ContainsKey(j)) { Rows4Col = new List <int>(); ColToRowExternal.Add(j, Rows4Col); } else { Rows4Col = ColToRowExternal[j]; } Rows4Col.Add(i + i0Row); } { List <int> ColForProc_proc; if (!ColForProc.ContainsKey(proc)) { ColForProc_proc = new List <int>(); ColForProc.Add(proc, ColForProc_proc); } else { ColForProc_proc = ColForProc[proc]; } if (!ColForProc_proc.Contains(ColIndex)) { ColForProc_proc.Add(ColIndex); } } } } } // communicate // =========== //SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw.MPI_COMM_WORLD); //{ // foreach (int proc in ColToRowExternal.Keys) { // sms.SetCommPath(proc); // } // sms.CommitCommPaths(); // // send // foreach (int proc in ColToRowExternal.Keys) { // SortedDictionary<int, List<int>> ColToRowExt_proc = ColToRowExternal[proc]; // SortedList _ColToRowExt_proc = new SortedList(); // foreach (int iCol in ColToRowExt_proc.Keys) // _ColToRowExt_proc.Add(iCol, ColToRowExt_proc[iCol].ToArray()); // } // // receive // int p; SortedList rcv; // sms.GetNext(out p, out rcv); // while (rcv != null) { // foreach (int col in rcv.Keys) { // int[] rowList = (int[])rcv[col]; // ColToRowLocal[col].AddRange(rowList); // } // sms.GetNext(out p, out rcv); // } //} //sms.Dispose(); // build 'ColProcessors' // ===================== //ColProcessors = new List<int>[ColToRowLocal.Length]; //for (int j = 0; j < ColToRowLocal.Length; j++) { // List<int> mpiRank = null; // foreach (int rowind in ColToRowLocal[j]) { // int riloc = rowind - i0Row; // if (riloc < 0 || riloc >= I) { // if (mpiRank == null) // mpiRank = new List<int>(); // mpiRank.Add(M.RowPartiton.FindProcess(rowind)); // } // ColProcessors[j] = mpiRank; // } //} // communicate: build 'ColProcessors' // ================================== { ColProcessors = new List <int> [ColPart.LocalLength]; SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD); sms.SetCommPathsAndCommit(ColForProc.Keys); foreach (int proc in ColForProc.Keys) { sms.Transmit(proc, ColForProc[proc].ToArray()); } int i0Loc = (int)ColPart.i0; int rcvproc; int[] ColIndices; while (sms.GetNext(out rcvproc, out ColIndices)) { int Rank; { csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out Rank); //Console.WriteLine("P# " + Rank + ": receiving from P# " + rcvproc); } foreach (int ColInd in ColIndices) { int localColInd = ColInd - i0Loc; if (localColInd < 0 || localColInd >= ColPart.LocalLength) { throw new IndexOutOfRangeException("internal error"); } if (ColProcessors[localColInd] == null) { ColProcessors[localColInd] = new List <int>(); } if (ColProcessors[localColInd].Contains(rcvproc)) { throw new ApplicationException("internal error."); } ColProcessors[localColInd].Add(rcvproc); } } sms.Dispose(); } if (ExtCol != null) { var send = new Dictionary <int, List <Tuple <int, List <int> > > >(); int myRank = M.RowPartitioning.MpiRank; foreach (var kv in ExtCol) { int rank = kv.Key; int[] ColIdx = kv.Value; var sendToRank = new List <Tuple <int, List <int> > >(); foreach (int iCol in ColIdx) { List <int> c2p = ColProcessors[iCol - i0Col]; var t = new Tuple <int, List <int> >(iCol, c2p != null ? new List <int>(c2p) : new List <int>()); t.Item2.Add(myRank); sendToRank.Add(t); } send.Add(rank, sendToRank); } var receive = SerialisationMessenger.ExchangeData(send, csMPI.Raw._COMM.WORLD); ColProcessorsExternal = new Dictionary <int, List <int> >(); foreach (var kv in receive) { var val = kv.Value; foreach (var t in val) { int iCol = t.Item1; List <int> ranks = t.Item2; int iMyRank = ranks.IndexOf(myRank); if (iMyRank >= 0) { ranks.RemoveAt(iMyRank); } ColProcessorsExternal.Add(t.Item1, t.Item2); Debug.Assert(this.ColPart.FindProcess(t.Item1) == kv.Key); } } #if DEBUG foreach (var procList in ColProcessorsExternal.Values) { Debug.Assert(procList.Contains(myRank) == false); } #endif } }