/// <summary>
/// Saves the matrix in a custom sparse text format;
/// Mainly for importing into MATLAB;
/// </summary>
/// <param name="path">Path to the text file</param>
/// <remarks>
/// MATLAB code for importing the matrix (save as file 'ReadMsr.m'):
/// <code>
/// function Mtx = ReadMsr(filename)
///
/// fid = fopen(filename);
/// % matrix dimensions
/// % -----------------
/// NoOfRows = fscanf(fid,'%d',1);
/// NoOfCols = fscanf(fid,'%d',1);
/// NonZeros = fscanf(fid,'%d',1);
/// cnt = 1;
/// % read row and column array
/// % -------------------------
/// iCol = zeros(NonZeros,1);
/// iRow = zeros(NonZeros,1);
/// entries = zeros(NonZeros,1);
/// l0 = 0;
/// str = char(zeros(1,6));
/// for i = 1:NoOfRows
/// NonZerosInRow = fscanf(fid,'%d',1);
/// if(l0 ~= NonZerosInRow)
/// str = char(zeros(1,NonZerosInRow*6));
/// for j = 1:NonZerosInRow
/// i0 = 1+(j-1)*6;
/// str(i0:i0+1) = '%f';
/// str(i0+3:i0+4) = '%f';
/// end
/// end
/// R = fscanf(fid,str,2*NonZerosInRow);
/// R2 = reshape(R',2,NonZerosInRow);
/// ind = cnt:(cnt+NonZerosInRow-1);
/// iCol(ind) = R2(1,:);
/// iRow(ind) = i;
/// entries(ind) = R2(2,:);
///
/// cnt = cnt + NonZerosInRow;
/// end
/// fclose(fid);
///
/// if (cnt-1) < NonZeros
/// iCol = iCol(1:(cnt-1),1);
/// iRow = iRow(1:(cnt-1),1);
/// entries = entries(1:(cnt-1),1);
/// end
///
/// % create sparse matrix
/// % --------------------
/// Mtx = sparse(iRow,iCol+1,entries,NoOfRows,NoOfCols,NonZeros);
///
/// </code>
/// </remarks>
/// <param name="M">
/// this pointer of extension method
/// </param>
static public void SaveToTextFileSparse(this IMutableMatrixEx M, string path)
{
using (new FuncTrace()) {
int rank, size;
csMPI.Raw.Comm_Rank(M.MPI_Comm, out rank);
csMPI.Raw.Comm_Size(M.MPI_Comm, out size);
SerialisationMessenger sms = new SerialisationMessenger(M.MPI_Comm);
int NoOfNonZeros = M.GetTotalNoOfNonZeros();
if (rank == 0)
{
sms.CommitCommPaths();
// receive data from other processors
MsrMatrix.MatrixEntry[][] entries = M.GetAllEntries();
if (size > 1)
{
Array.Resize(ref entries, (int)(M.RowPartitioning.TotalLength));
}
Helper rcvdata; int rcvRank;
while (sms.GetNext(out rcvRank, out rcvdata))
{
Array.Copy(rcvdata.entries, 0, entries, (int)M.RowPartitioning.GetI0Offest(rcvRank), rcvdata.entries.Length);
}
// open file
StreamWriter stw = new StreamWriter(path);
// serialize matrix data
stw.WriteLine(M.RowPartitioning.TotalLength); // number of rows
stw.WriteLine(M.NoOfCols); // number of columns
stw.WriteLine(NoOfNonZeros); // number of non-zero entries in Matrix (over all MPI-processors)
for (int i = 0; i < entries.Length; i++)
{
MsrMatrix.MatrixEntry[] row = entries[i];
int NonZPRow = 0;
foreach (MsrMatrix.MatrixEntry e in row)
{
if (e.ColIndex >= 0 && e.Value != 0.0)
{
NonZPRow++;
}
}
stw.Write(NonZPRow);
stw.Write(" ");
foreach (MsrMatrix.MatrixEntry e in row)
{
if (e.ColIndex >= 0 && e.Value != 0.0)
{
stw.Write(e.ColIndex);
stw.Write(" ");
stw.Write(e.Value.ToString("E16", NumberFormatInfo.InvariantInfo));
stw.Write(" ");
}
}
stw.WriteLine();
}
// finalize
stw.Flush();
stw.Close();
}
else
{
sms.SetCommPath(0);
sms.CommitCommPaths();
var entries = M.GetAllEntries();
var c = new Helper();
c.entries = entries;
sms.Transmitt(0, c);
MsrMatrix.MatrixEntry[][] dummy; int dummy_;
if (sms.GetNext <MsrMatrix.MatrixEntry[][]>(out dummy_, out dummy))
{
throw new ApplicationException("error in app");
}
}
sms.Dispose();
}
}
/// <summary>
/// matrix assembly; must be called by each implementation,
/// </summary>
/// <param name="M"></param>
protected void PackMatrix(IMutableMatrixEx M)
{
ilPSP.MPICollectiveWatchDog.Watch();
IPartitioning rp = M.RowPartitioning;
IPartitioning cp = m_ColPart;
// define Comm List
// ================
SortedDictionary <int, List <int> > CommLists = new SortedDictionary <int, List <int> >();
// keys: processor rank p
// values: List of global indices, which processor p needs to send to this processor
int Lr;
double[] val = null;
int[] col = null;
int L = rp.LocalLength;
int i0 = (int)rp.i0;
for (int iLoc = 0; iLoc < L; iLoc++) // loop over all matrix rows...
{
int iGlob = i0 + iLoc;
//MsrMatrix.MatrixEntry[] row = (asMsr==null) ? M.GetRow(iGlob) : asMsr.GetRowShallow(iGlob);
Lr = M.GetOccupiedColumnIndices(iGlob, ref col);
for (int j = 0; j < Lr; j++) // loop over all nonzero entries in row 'iGlob'
{
int jGlob = col[j];
if (cp.i0 <= jGlob && jGlob < (cp.i0 + cp.LocalLength))
{
// Entry on current processor
}
else
{
int proc = cp.FindProcess(jGlob);
// Entry on Processor proc
if (!CommLists.ContainsKey(proc))
{
CommLists.Add(proc, new List <int>());
}
List <int> CommList_proc = CommLists[proc];
if (!CommList_proc.Contains(jGlob)) // a lot of room for optimization
{
CommList_proc.Add(jGlob);
}
}
}
}
// sort com list
// =============
{
foreach (List <int> cl in CommLists.Values)
{
cl.Sort();
}
}
// define matrix
// =============
{
TempCSR intTmp = new TempCSR();
SortedDictionary <int, ExternalTmp> extTmp = new SortedDictionary <int, ExternalTmp>();
foreach (int proc in CommLists.Keys)
{
extTmp.Add(proc, new ExternalTmp());
}
for (int iLoc = 0; iLoc < L; iLoc++)
{
int iGlob = i0 + iLoc;
Lr = M.GetRow(iGlob, ref col, ref val);
for (int j = 0; j < Lr; j++)
{
int jGlob = col[j];
double Value = val[j];
bool bIsDiag = (iGlob == jGlob);
if (cp.i0 <= jGlob && jGlob < (cp.i0 + cp.LocalLength))
{
// Entry on current processor
intTmp.AddEntry(jGlob - (int)cp.i0, Value, bIsDiag);
}
else
{
int proc = cp.FindProcess(jGlob);
// Entry on Processor proc
List <int> CommList_proc = CommLists[proc];
int jloc = CommList_proc.IndexOf(jGlob);
ExternalTmp et = extTmp[proc];
et.AddEntry(jloc, jGlob, Value);
}
}
intTmp.NextRow();
foreach (ExternalTmp et in extTmp.Values)
{
et.NextRow();
}
}
m_LocalMtx = AssembleFinalFormat(intTmp);
ExtMatrix = new Dictionary <int, External>();
foreach (int proc in extTmp.Keys)
{
ExtMatrix.Add(proc, extTmp[proc].GetFinalObj());
}
}
// send/receive & transform Comm lists
// ====================================
{
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
SortedDictionary <int, int[]> CommListsTo = new SortedDictionary <int, int[]>();
foreach (int proc in CommLists.Keys)
{
sms.SetCommPath(proc);
}
sms.CommitCommPaths();
foreach (int proc in CommLists.Keys)
{
sms.Transmitt(proc, CommLists[proc].ToArray());
}
int _proc;
int[] CommListReceived;
sms.GetNext(out _proc, out CommListReceived);
int Lcol = m_ColPart.LocalLength;
int i0col = (int)m_ColPart.i0;
while (CommListReceived != null)
{
// convert indices to local coordinates
for (int i = 0; i < CommListReceived.Length; i++)
{
CommListReceived[i] -= i0col;
// check:
if (CommListReceived[i] < 0 || CommListReceived[i] >= Lcol)
{
throw new ApplicationException("internal error: something wrong with received Comm List.");
}
}
CommListsTo.Add(_proc, CommListReceived);
sms.GetNext(out _proc, out CommListReceived);
}
sms.Dispose();
m_SpmvCommPattern = new SpmvCommPattern();
m_SpmvCommPattern.ComLists = CommListsTo;
}
// record the number of elements which we receive
// ==============================================
{
m_SpmvCommPattern.NoOfReceivedEntries = new Dictionary <int, int>();
foreach (int p in CommLists.Keys)
{
m_SpmvCommPattern.NoOfReceivedEntries.Add(p, CommLists[p].Count);
}
}
}
/// <summary>
/// detects how the MPI nodes are distributed over compute nodes (SMP nodes)
/// </summary>
private void SMPEvaluation()
{
//int ht = m_Context.IOMaster.tracer.EnterFunction("BoSSS.Foundation.Comm.DatabaseDriver.SMPEvaluation");
using (new FuncTrace()) {
ilPSP.MPICollectiveWatchDog.Watch(MPI.Wrappers.csMPI.Raw._COMM.WORLD);
// define SMP rank;
// for each MPI process, the SMP node index
// index: MPI rank; content: SMP rank;
int[] SMPRank = null;
int NoOfSMPs = -1;
{
// we are using the computer name to determine
// which MPI processes run on the same physical machine
// send host name to proc 0.
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
if (MyRank > 0)
{
sms.SetCommPath(0);
}
sms.CommitCommPaths();
if (MyRank > 0)
{
sms.Transmitt(0, m_hostname);
}
int recvRnk;
string nmn;
sms.GetNext(out recvRnk, out nmn);
if (MyRank == 0)
{
// receiving names form all processors
List <string> hosts_unique = new List <string>();
hosts_unique.Add(m_hostname);
string[] hosts = new string[Size];
SMPRank = new int[Size];
ArrayTools.SetAll(SMPRank, int.MinValue);
hosts[0] = m_hostname;
SMPRank[0] = hosts_unique.IndexOf(m_hostname);
while (nmn != null)
{
if (hosts[recvRnk] != null)
{
throw new ApplicationException("should not happen.");
}
hosts[recvRnk] = nmn;
int smpRnk = hosts_unique.IndexOf(nmn);
if (smpRnk < 0)
{
hosts_unique.Add(nmn);
smpRnk = hosts_unique.Count - 1;
}
SMPRank[recvRnk] = smpRnk;
sms.GetNext(out recvRnk, out nmn);
}
NoOfSMPs = hosts_unique.Count;
for (int i = 0; i < Size; i++)
{
if (hosts[i] == null || SMPRank[i] < 0)
{
throw new ApplicationException("fatal error in algorithm.");
}
}
}
else
{
// don't receive anything
if (nmn != null)
{
// fatal error in algorithm
throw new ApplicationException("ha?");
}
}
sms.Dispose();
}
m_SMPSize = NoOfSMPs.MPIBroadcast(0, csMPI.Raw._COMM.WORLD);
m_SMPRanks = SMPRank.MPIBroadcast(0, csMPI.Raw._COMM.WORLD);
{
// number of MPI processes per SMP rank; index: SMP rank
m_MPIProcessesPerSMP = new int[m_SMPSize];
int[] _MPIProcessesPerSMP = new int[m_SMPSize];
_MPIProcessesPerSMP[m_SMPRanks[m_MyRank]]++;
unsafe
{
fixed(int *pSnd = &_MPIProcessesPerSMP[0], pRcv = &m_MPIProcessesPerSMP[0])
{
csMPI.Raw.Allreduce((IntPtr)pSnd, (IntPtr)pRcv, m_SMPSize, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.SUM, csMPI.Raw._COMM.WORLD);
}
}
}
//m_Context.IOMaster.tracer.LeaveFunction(ht);
}
}
/// <summary>
/// MPI update of a bit-array
/// </summary>
/// <param name="b"></param>
/// <param name="GridData"></param>
static public void MPIExchange(this BitArray b, IGridData GridData)
{
if (b.Length != GridData.iLogicalCells.NoOfCells)
{
throw new ArgumentException("length must be equal to number of cells.", "b");
}
if (GridData.CellPartitioning.MpiSize > 1)
{
// external
{
int rank, size;
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank);
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out size);
// setup messenger
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
sms.SetCommPathsAndCommit(GridData.iParallel.ProcessesToSendTo);
// send data
for (int p = 0; p < size; p++)
{
int[] sendlist = GridData.iParallel.SendCommLists[p];
if (sendlist == null)
{
continue;
}
int L = sendlist.Length;
System.Collections.BitArray packet_for_p = new System.Collections.BitArray(L, false);
for (int l = 0; l < L; l++)
{
packet_for_p[l] = b[sendlist[l]];
}
sms.Transmitt(p, packet_for_p);
}
// receive data
System.Collections.BitArray rcv_dat;
int rcv_rank;
while (sms.GetNext(out rcv_rank, out rcv_dat))
{
int insertAt = GridData.iParallel.RcvCommListsInsertIndex[rcv_rank];
if (GridData.iParallel.RcvCommListsNoOfItems[rcv_rank] != rcv_dat.Count)
{
throw new ApplicationException("internal error.");
}
int C = rcv_dat.Count;
for (int i = 0; i < C; i++)
{
b[insertAt + i] = rcv_dat[i];
}
}
// dispose
sms.Dispose();
}
}
}
/// <summary>
/// ctor.
/// </summary>
/// <param name="M"></param>
/// <param name="ExtCol">
/// key: processor rank 'p' <br/>
/// value: a list of column indices (within the local range of columns of <paramref name="M"/>),
/// which should be editable at rank 'p'.
/// </param>
public MsrExtMatrix(IMutableMatrixEx M, IDictionary <int, int[]> ExtCol)
{
this.ColPart = M.ColPartition;
int i0Row = (int)M.RowPartitioning.i0, I = M.RowPartitioning.LocalLength,
i0Col = (int)this.ColPart.i0, J = this.ColPart.LocalLength;
Mtx = M;
// init
// ====
ColToRowLocal = new List <int> [ColPart.LocalLength];
for (int j = 0; j < ColToRowLocal.Length; j++)
{
ColToRowLocal[j] = new List <int>();
}
// build Column to row - mapping
// =============================
ColToRowExternal = new Dictionary <int, List <int> >();
// key: global column index j, within the range of processor 'p'
// values: global row indices
SortedDictionary <int, List <int> > ColForProc = new SortedDictionary <int, List <int> >();
// key: MPI processor index 'p'
// values: a set of global column indices, within the range of processor 'p',
// that contain nonzero entries on this processor
int[] col = null;
int L;
// loop over all rows...
for (int i = 0; i < I; i++)
{
L = M.GetOccupiedColumnIndices(i + i0Row, ref col);
// loop over all nonzero entries in the row...
for (int l = 0; l < L; l++)
{
int ColIndex = col[l];
int localColInd = ColIndex - i0Col;
if (localColInd >= 0 && localColInd < J)
{
// column of 'entry' is within the local range of this processor
// + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColToRowLocal[localColInd].Add(i + i0Row);
}
else
{
// column of 'entry' belongs to external processor 'proc'
// + + + + + + + + + + + + + + + + + + + + + + + + + + + +
int proc = this.ColPart.FindProcess(ColIndex);
{
//SortedDictionary<int, List<int>> ColToRowExt_proc;
//if (!ColToRowExternal.ContainsKey(proc)) {
// ColToRowExt_proc = new SortedDictionary<int, List<int>>();
// ColToRowExternal.Add(proc, ColToRowExt_proc);
//} else {
// ColToRowExt_proc = ColToRowExternal[proc];
//}
int j = ColIndex;
List <int> Rows4Col;
if (!ColToRowExternal.ContainsKey(j))
{
Rows4Col = new List <int>();
ColToRowExternal.Add(j, Rows4Col);
}
else
{
Rows4Col = ColToRowExternal[j];
}
Rows4Col.Add(i + i0Row);
}
{
List <int> ColForProc_proc;
if (!ColForProc.ContainsKey(proc))
{
ColForProc_proc = new List <int>();
ColForProc.Add(proc, ColForProc_proc);
}
else
{
ColForProc_proc = ColForProc[proc];
}
if (!ColForProc_proc.Contains(ColIndex))
{
ColForProc_proc.Add(ColIndex);
}
}
}
}
}
// communicate
// ===========
//SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw.MPI_COMM_WORLD);
//{
// foreach (int proc in ColToRowExternal.Keys) {
// sms.SetCommPath(proc);
// }
// sms.CommitCommPaths();
// // send
// foreach (int proc in ColToRowExternal.Keys) {
// SortedDictionary<int, List<int>> ColToRowExt_proc = ColToRowExternal[proc];
// SortedList _ColToRowExt_proc = new SortedList();
// foreach (int iCol in ColToRowExt_proc.Keys)
// _ColToRowExt_proc.Add(iCol, ColToRowExt_proc[iCol].ToArray());
// }
// // receive
// int p; SortedList rcv;
// sms.GetNext(out p, out rcv);
// while (rcv != null) {
// foreach (int col in rcv.Keys) {
// int[] rowList = (int[])rcv[col];
// ColToRowLocal[col].AddRange(rowList);
// }
// sms.GetNext(out p, out rcv);
// }
//}
//sms.Dispose();
// build 'ColProcessors'
// =====================
//ColProcessors = new List<int>[ColToRowLocal.Length];
//for (int j = 0; j < ColToRowLocal.Length; j++) {
// List<int> mpiRank = null;
// foreach (int rowind in ColToRowLocal[j]) {
// int riloc = rowind - i0Row;
// if (riloc < 0 || riloc >= I) {
// if (mpiRank == null)
// mpiRank = new List<int>();
// mpiRank.Add(M.RowPartiton.FindProcess(rowind));
// }
// ColProcessors[j] = mpiRank;
// }
//}
// communicate: build 'ColProcessors'
// ==================================
{
ColProcessors = new List <int> [ColPart.LocalLength];
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
sms.SetCommPathsAndCommit(ColForProc.Keys);
foreach (int proc in ColForProc.Keys)
{
sms.Transmit(proc, ColForProc[proc].ToArray());
}
int i0Loc = (int)ColPart.i0;
int rcvproc; int[] ColIndices;
while (sms.GetNext(out rcvproc, out ColIndices))
{
int Rank;
{
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out Rank);
//Console.WriteLine("P# " + Rank + ": receiving from P# " + rcvproc);
}
foreach (int ColInd in ColIndices)
{
int localColInd = ColInd - i0Loc;
if (localColInd < 0 || localColInd >= ColPart.LocalLength)
{
throw new IndexOutOfRangeException("internal error");
}
if (ColProcessors[localColInd] == null)
{
ColProcessors[localColInd] = new List <int>();
}
if (ColProcessors[localColInd].Contains(rcvproc))
{
throw new ApplicationException("internal error.");
}
ColProcessors[localColInd].Add(rcvproc);
}
}
sms.Dispose();
}
if (ExtCol != null)
{
var send = new Dictionary <int, List <Tuple <int, List <int> > > >();
int myRank = M.RowPartitioning.MpiRank;
foreach (var kv in ExtCol)
{
int rank = kv.Key;
int[] ColIdx = kv.Value;
var sendToRank = new List <Tuple <int, List <int> > >();
foreach (int iCol in ColIdx)
{
List <int> c2p = ColProcessors[iCol - i0Col];
var t = new Tuple <int, List <int> >(iCol, c2p != null ? new List <int>(c2p) : new List <int>());
t.Item2.Add(myRank);
sendToRank.Add(t);
}
send.Add(rank, sendToRank);
}
var receive = SerialisationMessenger.ExchangeData(send, csMPI.Raw._COMM.WORLD);
ColProcessorsExternal = new Dictionary <int, List <int> >();
foreach (var kv in receive)
{
var val = kv.Value;
foreach (var t in val)
{
int iCol = t.Item1;
List <int> ranks = t.Item2;
int iMyRank = ranks.IndexOf(myRank);
if (iMyRank >= 0)
{
ranks.RemoveAt(iMyRank);
}
ColProcessorsExternal.Add(t.Item1, t.Item2);
Debug.Assert(this.ColPart.FindProcess(t.Item1) == kv.Key);
}
}
#if DEBUG
foreach (var procList in ColProcessorsExternal.Values)
{
Debug.Assert(procList.Contains(myRank) == false);
}
#endif
}
}
