/// <summary> /// finds maximum and minimum entry -- within the part that is stored on the local MPI process -- /// of some matrix. /// </summary> /// <param name="M">input; the matrix to work on</param> /// <param name="Min">output: the minimum entry of <paramref name="M"/> within the local MPI process</param> /// <param name="MinRow">output: the row index, where <paramref name="Min"/> is located</param> /// <param name="MinCol">output: the column index, where <paramref name="Min"/> is located</param> /// <param name="Max">output: the maximum entry of <paramref name="M"/> within the local MPI process</param> /// <param name="MaxRow">output: the row index, where <paramref name="Max"/> is located</param> /// <param name="MaxCol">output: the column index, where <paramref name="Max"/> is located</param> public static void GetMinimumAndMaximum_MPILocal(this IMutableMatrixEx M, out double Min, out int MinRow, out int MinCol, out double Max, out int MaxRow, out int MaxCol) { Min = double.MaxValue; Max = double.MinValue; MinRow = int.MinValue; MinCol = int.MinValue; MaxRow = int.MinValue; MaxCol = int.MinValue; MsrMatrix _M = M as MsrMatrix; bool t = false; int I = M.RowPartitioning.LocalLength; int i0 = (int)M.RowPartitioning.i0; double[] val = null; int[] col = null; int L; for (int i = 0; i < I; i++) { int iRow = i + i0; L = M.GetRow(iRow, ref col, ref val); for (int l = 0; l < L; l++) { t = true; int ColIndex = col[l]; double Value = val[l]; if (Value > Max) { Max = Value; MaxRow = iRow; MaxCol = ColIndex; } if (Value < Min) { Min = Value; MinRow = iRow; MinCol = ColIndex; } } } if (!t) { // matrix is completely empty -> min and max is 0.0, 1st occurrence per def. @ (0,0) Min = 0; MinCol = 0; MinRow = 0; Max = 0; MaxCol = 0; MaxRow = 0; } }
/// <summary> /// accumulates a dense matrix <paramref name="FullMtx"/> to a sparse matrix /// -- certainly, only adviseable for small matrices. /// </summary> static public void AccDenseMatrix(this IMutableMatrixEx tis, double alpha, IMatrix FullMtx) { if (tis.RowPartitioning.LocalLength != FullMtx.NoOfRows) { throw new ArgumentException("Mismatch in number of rows."); } if (tis.ColPartition.TotalLength != FullMtx.NoOfCols) { throw new ArgumentException("Mismatch in number of columns."); } int i0 = tis.RowPartitioning.i0; int I = tis.RowPartitioning.LocalLength; int J = tis.ColPartition.TotalLength; int[] col = null; double[] val = null; for (int i = 0; i < I; i++) { int Lr = tis.GetRow(i + i0, ref col, ref val); var oldRow = new MsrMatrix.MatrixEntry[Lr]; for (int lr = 0; lr < Lr; lr++) { oldRow[i].m_ColIndex = col[lr]; oldRow[i].Value = val[lr]; } List <int> NewColIdx = new List <int>(J); List <double> NewVals = new List <double>(J); for (int j = 0; j < J; j++) { double FMij = FullMtx[i, j]; if (FMij != 0.0) { NewVals.Add(alpha * FMij); NewColIdx.Add(j); } } Array.Sort <MsrMatrix.MatrixEntry>(oldRow); int k1 = 0, k2 = 0, K1 = oldRow.Length, K2 = NewVals.Count; while (k1 < K1 && k2 < K2) { int j1 = oldRow[k1].m_ColIndex; int j2 = NewColIdx[k2]; if (j1 < 0) { // should also chrash in RELEASE, therefor -> Exception. throw new ApplicationException("expecting a row without un-allocated entries."); } if (j1 > j2) { // k2++; // new row neds to catch up } else if (j1 < j2) { k1++; } else { NewVals[k2] += oldRow[k1].Value; k1++; k2++; } } tis.SetValues(i + i0, NewColIdx.ToArray(), NewVals.ToArray()); } }
/// <summary> /// only for debugging and testing; converts the matrix to a full matrix; /// when running in parallel, the matrix is collected on process 0. /// </summary> /// <returns> /// null on all MPI processes, except on rank 0; /// </returns> static public MultidimensionalArray ToFullMatrixOnProc0(this IMutableMatrixEx tis) { var comm = tis.MPI_Comm; int Rank = tis.RowPartitioning.MpiRank; int Size = tis.RowPartitioning.MpiSize; double[,] ret = null; if (Rank == 0) { ret = new double[tis.NoOfRows, tis.NoOfCols]; } SerialisationMessenger sms = new SerialisationMessenger(comm); if (Rank > 0) { sms.SetCommPath(0); } sms.CommitCommPaths(); Tuple <int, int[], double[]>[] data; { int L = tis.RowPartitioning.LocalLength; data = new Tuple <int, int[], double[]> [L]; int i0 = (int)tis.RowPartitioning.i0; for (int i = 0; i < L; i++) { double[] val = null; // this mem. must be inside the loop/allocated for every i and cannot be reused because we need all rows later. int[] col = null; int Lr = tis.GetRow(i + i0, ref col, ref val); data[i] = new Tuple <int, int[], double[]>(Lr, col, val); } } if (Rank > 0) { sms.Transmit(0, data); } int rcvProc = 0; if (Rank == 0) { do { int i0 = (int)tis.RowPartitioning.GetI0Offest(rcvProc); if (data.Length != tis.RowPartitioning.GetLocalLength(rcvProc)) { throw new ApplicationException("internal error"); } for (int i = 0; i < data.Length; i++) { //foreach (MsrMatrix.MatrixEntry entry in data[i]) { // if (entry.m_ColIndex >= 0) // ret[i + i0, entry.m_ColIndex] = entry.Value; //} int Lr = data[i].Item1; int[] col = data[i].Item2; double[] val = data[i].Item3; for (int lr = 0; lr < Lr; lr++) { ret[i + i0, col[lr]] = val[lr]; } } } while (sms.GetNext(out rcvProc, out data)); } else { if (sms.GetNext(out rcvProc, out data)) { throw new ApplicationException("internal error"); } } if (Rank == 0) { var _ret = MultidimensionalArray.Create(ret.GetLength(0), ret.GetLength(1)); for (int i = 0; i < _ret.NoOfRows; i++) { for (int j = 0; j < _ret.NoOfCols; j++) { _ret[i, j] = ret[i, j]; } } return(_ret); } else { return(null); } }
/// <summary> /// Saves the matrix in a custom sparse text format; /// Mainly for importing into MATLAB; /// </summary> /// <param name="path">Path to the text file</param> /// <remarks> /// MATLAB code for importing the matrix (save as file 'ReadMsr.m'): /// <code> /// function Mtx = ReadMsr(filename) /// /// fid = fopen(filename); /// % matrix dimensions /// % ----------------- /// NoOfRows = fscanf(fid,'%d',1); /// NoOfCols = fscanf(fid,'%d',1); /// NonZeros = fscanf(fid,'%d',1); /// cnt = 1; /// % read row and column array /// % ------------------------- /// iCol = zeros(NonZeros,1); /// iRow = zeros(NonZeros,1); /// entries = zeros(NonZeros,1); /// l0 = 0; /// str = char(zeros(1,6)); /// for i = 1:NoOfRows /// NonZerosInRow = fscanf(fid,'%d',1); /// if(l0 ~= NonZerosInRow) /// str = char(zeros(1,NonZerosInRow*6)); /// for j = 1:NonZerosInRow /// i0 = 1+(j-1)*6; /// str(i0:i0+2) = '%f '; /// str(i0+3:i0+5) = '%f '; /// end /// end /// R = fscanf(fid,str,2*NonZerosInRow); /// R2 = reshape(R',2,NonZerosInRow); /// ind = cnt:(cnt+NonZerosInRow-1); /// iCol(ind) = R2(1,:); /// iRow(ind) = i; /// entries(ind) = R2(2,:); /// /// cnt = cnt + NonZerosInRow; /// end /// fclose(fid); /// /// if (cnt-1) < NonZeros /// iCol = iCol(1:(cnt-1),1); /// iRow = iRow(1:(cnt-1),1); /// entries = entries(1:(cnt-1),1); /// end /// /// % create sparse matrix /// % -------------------- /// Mtx = sparse(iRow,iCol+1,entries,NoOfRows,NoOfCols,NonZeros); /// /// </code> /// </remarks> /// <param name="M"> /// this pointer of extension method /// </param> static public void SaveToTextFileSparse(this IMutableMatrixEx M, string path) { using (new FuncTrace()) { int rank, size; csMPI.Raw.Comm_Rank(M.MPI_Comm, out rank); csMPI.Raw.Comm_Size(M.MPI_Comm, out size); SerialisationMessenger sms = new SerialisationMessenger(M.MPI_Comm); int NoOfNonZeros = M.GetTotalNoOfNonZeros(); if (rank == 0) { sms.CommitCommPaths(); // receive data from other processors MsrMatrix.MatrixEntry[][] entries = M.GetAllEntries(); if (size > 1) { Array.Resize(ref entries, (int)(M.RowPartitioning.TotalLength)); } Helper rcvdata; int rcvRank; while (sms.GetNext(out rcvRank, out rcvdata)) { Array.Copy(rcvdata.entries, 0, entries, (int)M.RowPartitioning.GetI0Offest(rcvRank), rcvdata.entries.Length); } // open file StreamWriter stw = new StreamWriter(path); // serialize matrix data stw.WriteLine(M.RowPartitioning.TotalLength); // number of rows stw.WriteLine(M.NoOfCols); // number of columns stw.WriteLine(NoOfNonZeros); // number of non-zero entries in Matrix (over all MPI-processors) for (int i = 0; i < entries.Length; i++) { MsrMatrix.MatrixEntry[] row = entries[i]; int NonZPRow = 0; foreach (MsrMatrix.MatrixEntry e in row) { if (e.ColIndex >= 0 && e.Value != 0.0) { NonZPRow++; } } stw.Write(NonZPRow); stw.Write(" "); foreach (MsrMatrix.MatrixEntry e in row) { if (e.ColIndex >= 0 && e.Value != 0.0) { stw.Write(e.ColIndex); stw.Write(" "); stw.Write(e.Value.ToString("E16", NumberFormatInfo.InvariantInfo)); stw.Write(" "); } } stw.WriteLine(); } // finalize stw.Flush(); stw.Close(); } else { sms.SetCommPath(0); sms.CommitCommPaths(); var entries = M.GetAllEntries(); var c = new Helper(); c.entries = entries; sms.Transmit(0, c); MsrMatrix.MatrixEntry[][] dummy; int dummy_; if (sms.GetNext <MsrMatrix.MatrixEntry[][]>(out dummy_, out dummy)) { throw new ApplicationException("error in app"); } } sms.Dispose(); } }
/// <summary> /// ctor. /// </summary> /// <param name="M"></param> /// <param name="ExtCol"> /// key: processor rank 'p' <br/> /// value: a list of column indices (within the local range of columns of <paramref name="M"/>), /// which should be editable at rank 'p'. /// </param> public MsrExtMatrix(IMutableMatrixEx M, IDictionary <int, int[]> ExtCol) { this.ColPart = M.ColPartition; int i0Row = (int)M.RowPartitioning.i0, I = M.RowPartitioning.LocalLength, i0Col = (int)this.ColPart.i0, J = this.ColPart.LocalLength; Mtx = M; // init // ==== ColToRowLocal = new List <int> [ColPart.LocalLength]; for (int j = 0; j < ColToRowLocal.Length; j++) { ColToRowLocal[j] = new List <int>(); } // build Column to row - mapping // ============================= ColToRowExternal = new Dictionary <int, List <int> >(); // key: global column index j, within the range of processor 'p' // values: global row indices SortedDictionary <int, List <int> > ColForProc = new SortedDictionary <int, List <int> >(); // key: MPI processor index 'p' // values: a set of global column indices, within the range of processor 'p', // that contain nonzero entries on this processor int[] col = null; int L; // loop over all rows... for (int i = 0; i < I; i++) { L = M.GetOccupiedColumnIndices(i + i0Row, ref col); // loop over all nonzero entries in the row... for (int l = 0; l < L; l++) { int ColIndex = col[l]; int localColInd = ColIndex - i0Col; if (localColInd >= 0 && localColInd < J) { // column of 'entry' is within the local range of this processor // + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ColToRowLocal[localColInd].Add(i + i0Row); } else { // column of 'entry' belongs to external processor 'proc' // + + + + + + + + + + + + + + + + + + + + + + + + + + + + int proc = this.ColPart.FindProcess(ColIndex); { //SortedDictionary<int, List<int>> ColToRowExt_proc; //if (!ColToRowExternal.ContainsKey(proc)) { // ColToRowExt_proc = new SortedDictionary<int, List<int>>(); // ColToRowExternal.Add(proc, ColToRowExt_proc); //} else { // ColToRowExt_proc = ColToRowExternal[proc]; //} int j = ColIndex; List <int> Rows4Col; if (!ColToRowExternal.ContainsKey(j)) { Rows4Col = new List <int>(); ColToRowExternal.Add(j, Rows4Col); } else { Rows4Col = ColToRowExternal[j]; } Rows4Col.Add(i + i0Row); } { List <int> ColForProc_proc; if (!ColForProc.ContainsKey(proc)) { ColForProc_proc = new List <int>(); ColForProc.Add(proc, ColForProc_proc); } else { ColForProc_proc = ColForProc[proc]; } if (!ColForProc_proc.Contains(ColIndex)) { ColForProc_proc.Add(ColIndex); } } } } } // communicate // =========== //SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw.MPI_COMM_WORLD); //{ // foreach (int proc in ColToRowExternal.Keys) { // sms.SetCommPath(proc); // } // sms.CommitCommPaths(); // // send // foreach (int proc in ColToRowExternal.Keys) { // SortedDictionary<int, List<int>> ColToRowExt_proc = ColToRowExternal[proc]; // SortedList _ColToRowExt_proc = new SortedList(); // foreach (int iCol in ColToRowExt_proc.Keys) // _ColToRowExt_proc.Add(iCol, ColToRowExt_proc[iCol].ToArray()); // } // // receive // int p; SortedList rcv; // sms.GetNext(out p, out rcv); // while (rcv != null) { // foreach (int col in rcv.Keys) { // int[] rowList = (int[])rcv[col]; // ColToRowLocal[col].AddRange(rowList); // } // sms.GetNext(out p, out rcv); // } //} //sms.Dispose(); // build 'ColProcessors' // ===================== //ColProcessors = new List<int>[ColToRowLocal.Length]; //for (int j = 0; j < ColToRowLocal.Length; j++) { // List<int> mpiRank = null; // foreach (int rowind in ColToRowLocal[j]) { // int riloc = rowind - i0Row; // if (riloc < 0 || riloc >= I) { // if (mpiRank == null) // mpiRank = new List<int>(); // mpiRank.Add(M.RowPartiton.FindProcess(rowind)); // } // ColProcessors[j] = mpiRank; // } //} // communicate: build 'ColProcessors' // ================================== { ColProcessors = new List <int> [ColPart.LocalLength]; SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD); sms.SetCommPathsAndCommit(ColForProc.Keys); foreach (int proc in ColForProc.Keys) { sms.Transmitt(proc, ColForProc[proc].ToArray()); } int i0Loc = (int)ColPart.i0; int rcvproc; int[] ColIndices; while (sms.GetNext(out rcvproc, out ColIndices)) { int Rank; { csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out Rank); //Console.WriteLine("P# " + Rank + ": receiving from P# " + rcvproc); } foreach (int ColInd in ColIndices) { int localColInd = ColInd - i0Loc; if (localColInd < 0 || localColInd >= ColPart.LocalLength) { throw new IndexOutOfRangeException("internal error"); } if (ColProcessors[localColInd] == null) { ColProcessors[localColInd] = new List <int>(); } if (ColProcessors[localColInd].Contains(rcvproc)) { throw new ApplicationException("internal error."); } ColProcessors[localColInd].Add(rcvproc); } } sms.Dispose(); } if (ExtCol != null) { var send = new Dictionary <int, List <Tuple <int, List <int> > > >(); int myRank = M.RowPartitioning.MpiRank; foreach (var kv in ExtCol) { int rank = kv.Key; int[] ColIdx = kv.Value; var sendToRank = new List <Tuple <int, List <int> > >(); foreach (int iCol in ColIdx) { List <int> c2p = ColProcessors[iCol - i0Col]; var t = new Tuple <int, List <int> >(iCol, c2p != null ? new List <int>(c2p) : new List <int>()); t.Item2.Add(myRank); sendToRank.Add(t); } send.Add(rank, sendToRank); } var receive = SerialisationMessenger.ExchangeData(send, csMPI.Raw._COMM.WORLD); ColProcessorsExternal = new Dictionary <int, List <int> >(); foreach (var kv in receive) { var val = kv.Value; foreach (var t in val) { int iCol = t.Item1; List <int> ranks = t.Item2; int iMyRank = ranks.IndexOf(myRank); if (iMyRank >= 0) { ranks.RemoveAt(iMyRank); } ColProcessorsExternal.Add(t.Item1, t.Item2); Debug.Assert(this.ColPart.FindProcess(t.Item1) == kv.Key); } } #if DEBUG foreach (var procList in ColProcessorsExternal.Values) { Debug.Assert(procList.Contains(myRank) == false); } #endif } }
/// <summary> /// constructor. /// </summary> /// <param name="M">matrix to manipulate</param> /// <param name="ExtCol"> /// key: processor rank 'p' <br/> /// value: a list of column indices (within the local range of columns of <paramref name="M"/>), /// which should be editable at rank 'p'. /// </param> public MatrixOp(IMutableMatrixEx M, IDictionary <int, int[]> ExtCol = null) { m_Matrix = new MsrExtMatrix(M, ExtCol); }
/// <summary> /// initializes this matrix to be a copy of <paramref name="mtx"/>; /// </summary> /// <param name="mtx"></param> public IJMatrix(IMutableMatrixEx mtx) { if (mtx.RowPartitioning.MPI_Comm != mtx.ColPartition.MPI_Comm) { throw new ArgumentException(); } if (mtx.RowPartitioning.IsMutable) { throw new ArgumentException(); } if (mtx.ColPartition.IsMutable) { throw new ArgumentException(); } m_RowPartition = mtx.RowPartitioning; m_ColPartition = mtx.ColPartition; if (mtx.NoOfRows != mtx.NoOfCols) { throw new ArgumentException("matrix must be quadratic.", "mtx"); } if (mtx.NoOfRows > (int.MaxValue - 2)) { throw new ApplicationException("unable to create HYPRE matrix: no. of matrix rows is larger than HYPRE index type (32 bit signed int);"); } if (mtx.NoOfCols > (int.MaxValue - 2)) { throw new ApplicationException("unable to create HYPRE matrix: no. of matrix columns is larger than HYPRE index type (32 bit signed int);"); } // matrix: init MPI_Comm comm = csMPI.Raw._COMM.WORLD; int ilower = (int)mtx.RowPartitioning.i0; int iupper = (int)ilower + mtx.RowPartitioning.LocalLength - 1; int jlower = (int)mtx.ColPartition.i0; int jupper = (int)jlower + mtx.ColPartition.LocalLength - 1; HypreException.Check(Wrappers.IJMatrix.Create(comm, ilower, iupper, jlower, jupper, out m_IJMatrix)); HypreException.Check(Wrappers.IJMatrix.SetObjectType(m_IJMatrix, Wrappers.Constants.HYPRE_PARCSR)); HypreException.Check(Wrappers.IJMatrix.Initialize(m_IJMatrix)); // matrix: set values, row by row ... int nrows, lmax = mtx.GetMaxNoOfNonZerosPerRow(); int[] rows = new int[1], cols = new int[lmax], ncols = new int[1]; double[] values = new double[lmax]; int LR; int[] col = null; double[] val = null; for (int i = 0; i < mtx.RowPartitioning.LocalLength; i++) { int iRowGlob = i + mtx.RowPartitioning.i0; LR = mtx.GetRow(iRowGlob, ref col, ref val); nrows = 1; rows[0] = iRowGlob; ncols[0] = LR; int cnt = 0; for (int j = 0; j < LR; j++) { if (val[j] != 0.0) { cols[cnt] = col[j]; values[cnt] = val[j]; cnt++; } } if (cnt <= 0) { throw new ArgumentException(string.Format("Zero matrix row detected (local row index: {0}, global row index: {1}).", i, iRowGlob)); } HypreException.Check(Wrappers.IJMatrix.SetValues(m_IJMatrix, nrows, ncols, rows, cols, values)); } // matrix: assembly HypreException.Check(Wrappers.IJMatrix.Assemble(m_IJMatrix)); HypreException.Check(Wrappers.IJMatrix.GetObject(m_IJMatrix, out m_ParCSR_matrix)); }
/// <summary> /// modifies a matrix <paramref name="Mtx"/> and a right-hand-side <paramref name="rhs"/> /// in order to fix the pressure at some reference point /// </summary> /// <param name="map">row mapping for <paramref name="Mtx"/> as well as <paramref name="rhs"/></param> /// <param name="iVar">the index of the pressure variable in the mapping <paramref name="map"/>.</param> /// <param name="LsTrk"></param> /// <param name="Mtx"></param> /// <param name="rhs"></param> static public void SetPressureReferencePoint <T>(UnsetteledCoordinateMapping map, int iVar, LevelSetTracker LsTrk, IMutableMatrixEx Mtx, T rhs) where T : IList <double> { using (new FuncTrace()) { var GridDat = map.GridDat; if (rhs.Count != map.LocalLength) { throw new ArgumentException(); } if (!Mtx.RowPartitioning.EqualsPartition(map) || !Mtx.ColPartition.EqualsPartition(map)) { throw new ArgumentException(); } Basis PressureBasis = (Basis)map.BasisS[iVar]; int D = GridDat.SpatialDimension; long GlobalID, GlobalCellIndex; bool IsInside, onthisProc; GridDat.LocatePoint(new double[D], out GlobalID, out GlobalCellIndex, out IsInside, out onthisProc, LsTrk != null ? LsTrk.Regions.GetCutCellSubGrid().VolumeMask.Complement() : null); int iRowGl = -111; if (onthisProc) { //int jCell = (int)GlobalCellIndex - GridDat.CellPartitioning.i0; //NodeSet CenterNode = new NodeSet(GridDat.iGeomCells.GetRefElement(jCell), new double[D]); //MultidimensionalArray LevSetValues = LsTrk.DataHistories[0].Current.GetLevSetValues(CenterNode, jCell, 1); ; //MultidimensionalArray CenterNodeGlobal = MultidimensionalArray.Create(1, D); //GridDat.TransformLocal2Global(CenterNode, CenterNodeGlobal, jCell); //Console.WriteLine("Pressure Ref Point @( {0:0.###E-00} | {1:0.###E-00} )", CenterNodeGlobal[0,0], CenterNodeGlobal[0,1]); //LevelSetSignCode scode = LevelSetSignCode.ComputeLevelSetBytecode(LevSetValues[0, 0]); //ReducedRegionCode rrc; //int No = LsTrk.Regions.GetNoOfSpecies(jCell, out rrc); //int iSpc = LsTrk.GetSpeciesIndex(rrc, scode); iRowGl = (int)map.GlobalUniqueCoordinateIndex_FromGlobal(iVar, GlobalCellIndex, 0); } iRowGl = iRowGl.MPIMax(); // clear row // --------- if (onthisProc) { // ref. cell is on local MPI process int jCell = (int)GlobalCellIndex - GridDat.CellPartitioning.i0; //ReducedRegionCode rrc; //int NoOfSpc = LsTrk.Regions.GetNoOfSpecies(jCell, out rrc); // set matrix row to identity Mtx.ClearRow(iRowGl); Mtx.SetDiagonalElement(iRowGl, 1.0); // clear RHS int iRow = iRowGl - Mtx.RowPartitioning.i0; rhs[iRow] = 0; } // clear column // ------------ { for (int i = Mtx.RowPartitioning.i0; i < Mtx.RowPartitioning.iE; i++) { if (i != iRowGl) { Mtx[i, iRowGl] = 0; } } } } }