/// <summary> /// Saves the matrix in a custom sparse text format; /// Mainly for importing into MATLAB; /// </summary> /// <param name="path">Path to the text file</param> /// <remarks> /// MATLAB code for importing the matrix (save as file 'ReadMsr.m'): /// <code> /// function Mtx = ReadMsr(filename) /// /// fid = fopen(filename); /// % matrix dimensions /// % ----------------- /// NoOfRows = fscanf(fid,'%d',1); /// NoOfCols = fscanf(fid,'%d',1); /// NonZeros = fscanf(fid,'%d',1); /// cnt = 1; /// % read row and column array /// % ------------------------- /// iCol = zeros(NonZeros,1); /// iRow = zeros(NonZeros,1); /// entries = zeros(NonZeros,1); /// l0 = 0; /// str = char(zeros(1,6)); /// for i = 1:NoOfRows /// NonZerosInRow = fscanf(fid,'%d',1); /// if(l0 ~= NonZerosInRow) /// str = char(zeros(1,NonZerosInRow*6)); /// for j = 1:NonZerosInRow /// i0 = 1+(j-1)*6; /// str(i0:i0+1) = '%f'; /// str(i0+3:i0+4) = '%f'; /// end /// end /// R = fscanf(fid,str,2*NonZerosInRow); /// R2 = reshape(R',2,NonZerosInRow); /// ind = cnt:(cnt+NonZerosInRow-1); /// iCol(ind) = R2(1,:); /// iRow(ind) = i; /// entries(ind) = R2(2,:); /// /// cnt = cnt + NonZerosInRow; /// end /// fclose(fid); /// /// if (cnt-1) < NonZeros /// iCol = iCol(1:(cnt-1),1); /// iRow = iRow(1:(cnt-1),1); /// entries = entries(1:(cnt-1),1); /// end /// /// % create sparse matrix /// % -------------------- /// Mtx = sparse(iRow,iCol+1,entries,NoOfRows,NoOfCols,NonZeros); /// /// </code> /// </remarks> /// <param name="M"> /// this pointer of extension method /// </param> static public void SaveToTextFileSparse(this IMutableMatrixEx M, string path) { using (new FuncTrace()) { int rank, size; csMPI.Raw.Comm_Rank(M.MPI_Comm, out rank); csMPI.Raw.Comm_Size(M.MPI_Comm, out size); SerialisationMessenger sms = new SerialisationMessenger(M.MPI_Comm); int NoOfNonZeros = M.GetTotalNoOfNonZeros(); if (rank == 0) { sms.CommitCommPaths(); // receive data from other processors MsrMatrix.MatrixEntry[][] entries = M.GetAllEntries(); if (size > 1) { Array.Resize(ref entries, (int)(M.RowPartitioning.TotalLength)); } Helper rcvdata; int rcvRank; while (sms.GetNext(out rcvRank, out rcvdata)) { Array.Copy(rcvdata.entries, 0, entries, (int)M.RowPartitioning.GetI0Offest(rcvRank), rcvdata.entries.Length); } // open file StreamWriter stw = new StreamWriter(path); // serialize matrix data stw.WriteLine(M.RowPartitioning.TotalLength); // number of rows stw.WriteLine(M.NoOfCols); // number of columns stw.WriteLine(NoOfNonZeros); // number of non-zero entries in Matrix (over all MPI-processors) for (int i = 0; i < entries.Length; i++) { MsrMatrix.MatrixEntry[] row = entries[i]; int NonZPRow = 0; foreach (MsrMatrix.MatrixEntry e in row) { if (e.ColIndex >= 0 && e.Value != 0.0) { NonZPRow++; } } stw.Write(NonZPRow); stw.Write(" "); foreach (MsrMatrix.MatrixEntry e in row) { if (e.ColIndex >= 0 && e.Value != 0.0) { stw.Write(e.ColIndex); stw.Write(" "); stw.Write(e.Value.ToString("E16", NumberFormatInfo.InvariantInfo)); stw.Write(" "); } } stw.WriteLine(); } // finalize stw.Flush(); stw.Close(); } else { sms.SetCommPath(0); sms.CommitCommPaths(); var entries = M.GetAllEntries(); var c = new Helper(); c.entries = entries; sms.Transmitt(0, c); MsrMatrix.MatrixEntry[][] dummy; int dummy_; if (sms.GetNext <MsrMatrix.MatrixEntry[][]>(out dummy_, out dummy)) { throw new ApplicationException("error in app"); } } sms.Dispose(); } }
/// <summary> /// matrix assembly; must be called by each implementation, /// </summary> /// <param name="M"></param> protected void PackMatrix(IMutableMatrixEx M) { ilPSP.MPICollectiveWatchDog.Watch(); IPartitioning rp = M.RowPartitioning; IPartitioning cp = m_ColPart; // define Comm List // ================ SortedDictionary <int, List <int> > CommLists = new SortedDictionary <int, List <int> >(); // keys: processor rank p // values: List of global indices, which processor p needs to send to this processor int Lr; double[] val = null; int[] col = null; int L = rp.LocalLength; int i0 = (int)rp.i0; for (int iLoc = 0; iLoc < L; iLoc++) // loop over all matrix rows... { int iGlob = i0 + iLoc; //MsrMatrix.MatrixEntry[] row = (asMsr==null) ? M.GetRow(iGlob) : asMsr.GetRowShallow(iGlob); Lr = M.GetOccupiedColumnIndices(iGlob, ref col); for (int j = 0; j < Lr; j++) // loop over all nonzero entries in row 'iGlob' { int jGlob = col[j]; if (cp.i0 <= jGlob && jGlob < (cp.i0 + cp.LocalLength)) { // Entry on current processor } else { int proc = cp.FindProcess(jGlob); // Entry on Processor proc if (!CommLists.ContainsKey(proc)) { CommLists.Add(proc, new List <int>()); } List <int> CommList_proc = CommLists[proc]; if (!CommList_proc.Contains(jGlob)) // a lot of room for optimization { CommList_proc.Add(jGlob); } } } } // sort com list // ============= { foreach (List <int> cl in CommLists.Values) { cl.Sort(); } } // define matrix // ============= { TempCSR intTmp = new TempCSR(); SortedDictionary <int, ExternalTmp> extTmp = new SortedDictionary <int, ExternalTmp>(); foreach (int proc in CommLists.Keys) { extTmp.Add(proc, new ExternalTmp()); } for (int iLoc = 0; iLoc < L; iLoc++) { int iGlob = i0 + iLoc; Lr = M.GetRow(iGlob, ref col, ref val); for (int j = 0; j < Lr; j++) { int jGlob = col[j]; double Value = val[j]; bool bIsDiag = (iGlob == jGlob); if (cp.i0 <= jGlob && jGlob < (cp.i0 + cp.LocalLength)) { // Entry on current processor intTmp.AddEntry(jGlob - (int)cp.i0, Value, bIsDiag); } else { int proc = cp.FindProcess(jGlob); // Entry on Processor proc List <int> CommList_proc = CommLists[proc]; int jloc = CommList_proc.IndexOf(jGlob); ExternalTmp et = extTmp[proc]; et.AddEntry(jloc, jGlob, Value); } } intTmp.NextRow(); foreach (ExternalTmp et in extTmp.Values) { et.NextRow(); } } m_LocalMtx = AssembleFinalFormat(intTmp); ExtMatrix = new Dictionary <int, External>(); foreach (int proc in extTmp.Keys) { ExtMatrix.Add(proc, extTmp[proc].GetFinalObj()); } } // send/receive & transform Comm lists // ==================================== { SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD); SortedDictionary <int, int[]> CommListsTo = new SortedDictionary <int, int[]>(); foreach (int proc in CommLists.Keys) { sms.SetCommPath(proc); } sms.CommitCommPaths(); foreach (int proc in CommLists.Keys) { sms.Transmitt(proc, CommLists[proc].ToArray()); } int _proc; int[] CommListReceived; sms.GetNext(out _proc, out CommListReceived); int Lcol = m_ColPart.LocalLength; int i0col = (int)m_ColPart.i0; while (CommListReceived != null) { // convert indices to local coordinates for (int i = 0; i < CommListReceived.Length; i++) { CommListReceived[i] -= i0col; // check: if (CommListReceived[i] < 0 || CommListReceived[i] >= Lcol) { throw new ApplicationException("internal error: something wrong with received Comm List."); } } CommListsTo.Add(_proc, CommListReceived); sms.GetNext(out _proc, out CommListReceived); } sms.Dispose(); m_SpmvCommPattern = new SpmvCommPattern(); m_SpmvCommPattern.ComLists = CommListsTo; } // record the number of elements which we receive // ============================================== { m_SpmvCommPattern.NoOfReceivedEntries = new Dictionary <int, int>(); foreach (int p in CommLists.Keys) { m_SpmvCommPattern.NoOfReceivedEntries.Add(p, CommLists[p].Count); } } }
/// <summary> /// detects how the MPI nodes are distributed over compute nodes (SMP nodes) /// </summary> private void SMPEvaluation() { //int ht = m_Context.IOMaster.tracer.EnterFunction("BoSSS.Foundation.Comm.DatabaseDriver.SMPEvaluation"); using (new FuncTrace()) { ilPSP.MPICollectiveWatchDog.Watch(MPI.Wrappers.csMPI.Raw._COMM.WORLD); // define SMP rank; // for each MPI process, the SMP node index // index: MPI rank; content: SMP rank; int[] SMPRank = null; int NoOfSMPs = -1; { // we are using the computer name to determine // which MPI processes run on the same physical machine // send host name to proc 0. SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD); if (MyRank > 0) { sms.SetCommPath(0); } sms.CommitCommPaths(); if (MyRank > 0) { sms.Transmitt(0, m_hostname); } int recvRnk; string nmn; sms.GetNext(out recvRnk, out nmn); if (MyRank == 0) { // receiving names form all processors List <string> hosts_unique = new List <string>(); hosts_unique.Add(m_hostname); string[] hosts = new string[Size]; SMPRank = new int[Size]; ArrayTools.SetAll(SMPRank, int.MinValue); hosts[0] = m_hostname; SMPRank[0] = hosts_unique.IndexOf(m_hostname); while (nmn != null) { if (hosts[recvRnk] != null) { throw new ApplicationException("should not happen."); } hosts[recvRnk] = nmn; int smpRnk = hosts_unique.IndexOf(nmn); if (smpRnk < 0) { hosts_unique.Add(nmn); smpRnk = hosts_unique.Count - 1; } SMPRank[recvRnk] = smpRnk; sms.GetNext(out recvRnk, out nmn); } NoOfSMPs = hosts_unique.Count; for (int i = 0; i < Size; i++) { if (hosts[i] == null || SMPRank[i] < 0) { throw new ApplicationException("fatal error in algorithm."); } } } else { // don't receive anything if (nmn != null) { // fatal error in algorithm throw new ApplicationException("ha?"); } } sms.Dispose(); } m_SMPSize = NoOfSMPs.MPIBroadcast(0, csMPI.Raw._COMM.WORLD); m_SMPRanks = SMPRank.MPIBroadcast(0, csMPI.Raw._COMM.WORLD); { // number of MPI processes per SMP rank; index: SMP rank m_MPIProcessesPerSMP = new int[m_SMPSize]; int[] _MPIProcessesPerSMP = new int[m_SMPSize]; _MPIProcessesPerSMP[m_SMPRanks[m_MyRank]]++; unsafe { fixed(int *pSnd = &_MPIProcessesPerSMP[0], pRcv = &m_MPIProcessesPerSMP[0]) { csMPI.Raw.Allreduce((IntPtr)pSnd, (IntPtr)pRcv, m_SMPSize, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.SUM, csMPI.Raw._COMM.WORLD); } } } //m_Context.IOMaster.tracer.LeaveFunction(ht); } }
/// <summary> /// MPI update of a bit-array /// </summary> /// <param name="b"></param> /// <param name="GridData"></param> static public void MPIExchange(this BitArray b, IGridData GridData) { if (b.Length != GridData.iLogicalCells.NoOfCells) { throw new ArgumentException("length must be equal to number of cells.", "b"); } if (GridData.CellPartitioning.MpiSize > 1) { // external { int rank, size; csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank); csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out size); // setup messenger SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD); sms.SetCommPathsAndCommit(GridData.iParallel.ProcessesToSendTo); // send data for (int p = 0; p < size; p++) { int[] sendlist = GridData.iParallel.SendCommLists[p]; if (sendlist == null) { continue; } int L = sendlist.Length; System.Collections.BitArray packet_for_p = new System.Collections.BitArray(L, false); for (int l = 0; l < L; l++) { packet_for_p[l] = b[sendlist[l]]; } sms.Transmitt(p, packet_for_p); } // receive data System.Collections.BitArray rcv_dat; int rcv_rank; while (sms.GetNext(out rcv_rank, out rcv_dat)) { int insertAt = GridData.iParallel.RcvCommListsInsertIndex[rcv_rank]; if (GridData.iParallel.RcvCommListsNoOfItems[rcv_rank] != rcv_dat.Count) { throw new ApplicationException("internal error."); } int C = rcv_dat.Count; for (int i = 0; i < C; i++) { b[insertAt + i] = rcv_dat[i]; } } // dispose sms.Dispose(); } } }
/// <summary> /// ctor. /// </summary> /// <param name="M"></param> /// <param name="ExtCol"> /// key: processor rank 'p' <br/> /// value: a list of column indices (within the local range of columns of <paramref name="M"/>), /// which should be editable at rank 'p'. /// </param> public MsrExtMatrix(IMutableMatrixEx M, IDictionary <int, int[]> ExtCol) { this.ColPart = M.ColPartition; int i0Row = (int)M.RowPartitioning.i0, I = M.RowPartitioning.LocalLength, i0Col = (int)this.ColPart.i0, J = this.ColPart.LocalLength; Mtx = M; // init // ==== ColToRowLocal = new List <int> [ColPart.LocalLength]; for (int j = 0; j < ColToRowLocal.Length; j++) { ColToRowLocal[j] = new List <int>(); } // build Column to row - mapping // ============================= ColToRowExternal = new Dictionary <int, List <int> >(); // key: global column index j, within the range of processor 'p' // values: global row indices SortedDictionary <int, List <int> > ColForProc = new SortedDictionary <int, List <int> >(); // key: MPI processor index 'p' // values: a set of global column indices, within the range of processor 'p', // that contain nonzero entries on this processor int[] col = null; int L; // loop over all rows... for (int i = 0; i < I; i++) { L = M.GetOccupiedColumnIndices(i + i0Row, ref col); // loop over all nonzero entries in the row... for (int l = 0; l < L; l++) { int ColIndex = col[l]; int localColInd = ColIndex - i0Col; if (localColInd >= 0 && localColInd < J) { // column of 'entry' is within the local range of this processor // + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ColToRowLocal[localColInd].Add(i + i0Row); } else { // column of 'entry' belongs to external processor 'proc' // + + + + + + + + + + + + + + + + + + + + + + + + + + + + int proc = this.ColPart.FindProcess(ColIndex); { //SortedDictionary<int, List<int>> ColToRowExt_proc; //if (!ColToRowExternal.ContainsKey(proc)) { // ColToRowExt_proc = new SortedDictionary<int, List<int>>(); // ColToRowExternal.Add(proc, ColToRowExt_proc); //} else { // ColToRowExt_proc = ColToRowExternal[proc]; //} int j = ColIndex; List <int> Rows4Col; if (!ColToRowExternal.ContainsKey(j)) { Rows4Col = new List <int>(); ColToRowExternal.Add(j, Rows4Col); } else { Rows4Col = ColToRowExternal[j]; } Rows4Col.Add(i + i0Row); } { List <int> ColForProc_proc; if (!ColForProc.ContainsKey(proc)) { ColForProc_proc = new List <int>(); ColForProc.Add(proc, ColForProc_proc); } else { ColForProc_proc = ColForProc[proc]; } if (!ColForProc_proc.Contains(ColIndex)) { ColForProc_proc.Add(ColIndex); } } } } } // communicate // =========== //SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw.MPI_COMM_WORLD); //{ // foreach (int proc in ColToRowExternal.Keys) { // sms.SetCommPath(proc); // } // sms.CommitCommPaths(); // // send // foreach (int proc in ColToRowExternal.Keys) { // SortedDictionary<int, List<int>> ColToRowExt_proc = ColToRowExternal[proc]; // SortedList _ColToRowExt_proc = new SortedList(); // foreach (int iCol in ColToRowExt_proc.Keys) // _ColToRowExt_proc.Add(iCol, ColToRowExt_proc[iCol].ToArray()); // } // // receive // int p; SortedList rcv; // sms.GetNext(out p, out rcv); // while (rcv != null) { // foreach (int col in rcv.Keys) { // int[] rowList = (int[])rcv[col]; // ColToRowLocal[col].AddRange(rowList); // } // sms.GetNext(out p, out rcv); // } //} //sms.Dispose(); // build 'ColProcessors' // ===================== //ColProcessors = new List<int>[ColToRowLocal.Length]; //for (int j = 0; j < ColToRowLocal.Length; j++) { // List<int> mpiRank = null; // foreach (int rowind in ColToRowLocal[j]) { // int riloc = rowind - i0Row; // if (riloc < 0 || riloc >= I) { // if (mpiRank == null) // mpiRank = new List<int>(); // mpiRank.Add(M.RowPartiton.FindProcess(rowind)); // } // ColProcessors[j] = mpiRank; // } //} // communicate: build 'ColProcessors' // ================================== { ColProcessors = new List <int> [ColPart.LocalLength]; SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD); sms.SetCommPathsAndCommit(ColForProc.Keys); foreach (int proc in ColForProc.Keys) { sms.Transmit(proc, ColForProc[proc].ToArray()); } int i0Loc = (int)ColPart.i0; int rcvproc; int[] ColIndices; while (sms.GetNext(out rcvproc, out ColIndices)) { int Rank; { csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out Rank); //Console.WriteLine("P# " + Rank + ": receiving from P# " + rcvproc); } foreach (int ColInd in ColIndices) { int localColInd = ColInd - i0Loc; if (localColInd < 0 || localColInd >= ColPart.LocalLength) { throw new IndexOutOfRangeException("internal error"); } if (ColProcessors[localColInd] == null) { ColProcessors[localColInd] = new List <int>(); } if (ColProcessors[localColInd].Contains(rcvproc)) { throw new ApplicationException("internal error."); } ColProcessors[localColInd].Add(rcvproc); } } sms.Dispose(); } if (ExtCol != null) { var send = new Dictionary <int, List <Tuple <int, List <int> > > >(); int myRank = M.RowPartitioning.MpiRank; foreach (var kv in ExtCol) { int rank = kv.Key; int[] ColIdx = kv.Value; var sendToRank = new List <Tuple <int, List <int> > >(); foreach (int iCol in ColIdx) { List <int> c2p = ColProcessors[iCol - i0Col]; var t = new Tuple <int, List <int> >(iCol, c2p != null ? new List <int>(c2p) : new List <int>()); t.Item2.Add(myRank); sendToRank.Add(t); } send.Add(rank, sendToRank); } var receive = SerialisationMessenger.ExchangeData(send, csMPI.Raw._COMM.WORLD); ColProcessorsExternal = new Dictionary <int, List <int> >(); foreach (var kv in receive) { var val = kv.Value; foreach (var t in val) { int iCol = t.Item1; List <int> ranks = t.Item2; int iMyRank = ranks.IndexOf(myRank); if (iMyRank >= 0) { ranks.RemoveAt(iMyRank); } ColProcessorsExternal.Add(t.Item1, t.Item2); Debug.Assert(this.ColPart.FindProcess(t.Item1) == kv.Key); } } #if DEBUG foreach (var procList in ColProcessorsExternal.Values) { Debug.Assert(procList.Contains(myRank) == false); } #endif } }