/// <summary>
/// Saves the matrix in a custom sparse text format;
/// Mainly for importing into MATLAB;
/// </summary>
/// <param name="path">Path to the text file</param>
/// <remarks>
/// MATLAB code for importing the matrix (save as file 'ReadMsr.m'):
/// <code>
/// function Mtx = ReadMsr(filename)
///
/// fid = fopen(filename);
/// % matrix dimensions
/// % -----------------
/// NoOfRows = fscanf(fid,'%d',1);
/// NoOfCols = fscanf(fid,'%d',1);
/// NonZeros = fscanf(fid,'%d',1);
/// cnt = 1;
/// % read row and column array
/// % -------------------------
/// iCol = zeros(NonZeros,1);
/// iRow = zeros(NonZeros,1);
/// entries = zeros(NonZeros,1);
/// l0 = 0;
/// str = char(zeros(1,6));
/// for i = 1:NoOfRows
/// NonZerosInRow = fscanf(fid,'%d',1);
/// if(l0 ~= NonZerosInRow)
/// str = char(zeros(1,NonZerosInRow*6));
/// for j = 1:NonZerosInRow
/// i0 = 1+(j-1)*6;
/// str(i0:i0+1) = '%f';
/// str(i0+3:i0+4) = '%f';
/// end
/// end
/// R = fscanf(fid,str,2*NonZerosInRow);
/// R2 = reshape(R',2,NonZerosInRow);
/// ind = cnt:(cnt+NonZerosInRow-1);
/// iCol(ind) = R2(1,:);
/// iRow(ind) = i;
/// entries(ind) = R2(2,:);
///
/// cnt = cnt + NonZerosInRow;
/// end
/// fclose(fid);
///
/// if (cnt-1) < NonZeros
/// iCol = iCol(1:(cnt-1),1);
/// iRow = iRow(1:(cnt-1),1);
/// entries = entries(1:(cnt-1),1);
/// end
///
/// % create sparse matrix
/// % --------------------
/// Mtx = sparse(iRow,iCol+1,entries,NoOfRows,NoOfCols,NonZeros);
///
/// </code>
/// </remarks>
/// <param name="M">
/// this pointer of extension method
/// </param>
static public void SaveToTextFileSparse(this IMutableMatrixEx M, string path)
{
using (new FuncTrace()) {
int rank, size;
csMPI.Raw.Comm_Rank(M.MPI_Comm, out rank);
csMPI.Raw.Comm_Size(M.MPI_Comm, out size);
SerialisationMessenger sms = new SerialisationMessenger(M.MPI_Comm);
int NoOfNonZeros = M.GetTotalNoOfNonZeros();
if (rank == 0)
{
sms.CommitCommPaths();
// receive data from other processors
MsrMatrix.MatrixEntry[][] entries = M.GetAllEntries();
if (size > 1)
{
Array.Resize(ref entries, (int)(M.RowPartitioning.TotalLength));
}
Helper rcvdata; int rcvRank;
while (sms.GetNext(out rcvRank, out rcvdata))
{
Array.Copy(rcvdata.entries, 0, entries, (int)M.RowPartitioning.GetI0Offest(rcvRank), rcvdata.entries.Length);
}
// open file
StreamWriter stw = new StreamWriter(path);
// serialize matrix data
stw.WriteLine(M.RowPartitioning.TotalLength); // number of rows
stw.WriteLine(M.NoOfCols); // number of columns
stw.WriteLine(NoOfNonZeros); // number of non-zero entries in Matrix (over all MPI-processors)
for (int i = 0; i < entries.Length; i++)
{
MsrMatrix.MatrixEntry[] row = entries[i];
int NonZPRow = 0;
foreach (MsrMatrix.MatrixEntry e in row)
{
if (e.ColIndex >= 0 && e.Value != 0.0)
{
NonZPRow++;
}
}
stw.Write(NonZPRow);
stw.Write(" ");
foreach (MsrMatrix.MatrixEntry e in row)
{
if (e.ColIndex >= 0 && e.Value != 0.0)
{
stw.Write(e.ColIndex);
stw.Write(" ");
stw.Write(e.Value.ToString("E16", NumberFormatInfo.InvariantInfo));
stw.Write(" ");
}
}
stw.WriteLine();
}
// finalize
stw.Flush();
stw.Close();
}
else
{
sms.SetCommPath(0);
sms.CommitCommPaths();
var entries = M.GetAllEntries();
var c = new Helper();
c.entries = entries;
sms.Transmitt(0, c);
MsrMatrix.MatrixEntry[][] dummy; int dummy_;
if (sms.GetNext <MsrMatrix.MatrixEntry[][]>(out dummy_, out dummy))
{
throw new ApplicationException("error in app");
}
}
sms.Dispose();
}
}
/// <summary>
/// only for debugging and testing; converts the matrix to a full matrix;
/// when running in parallel, the matrix is collected on process 0.
/// </summary>
/// <returns>
/// null on all MPI processes, except on rank 0;
/// </returns>
static public MultidimensionalArray ToFullMatrixOnProc0(this IMutableMatrixEx tis)
{
var comm = tis.MPI_Comm;
int Rank = tis.RowPartitioning.MpiRank;
int Size = tis.RowPartitioning.MpiSize;
double[,] ret = null;
if (Rank == 0)
{
ret = new double[tis.NoOfRows, tis.NoOfCols];
}
SerialisationMessenger sms = new SerialisationMessenger(comm);
if (Rank > 0)
{
sms.SetCommPath(0);
}
sms.CommitCommPaths();
Tuple <int, int[], double[]>[] data;
{
int L = tis.RowPartitioning.LocalLength;
data = new Tuple <int, int[], double[]> [L];
int i0 = (int)tis.RowPartitioning.i0;
for (int i = 0; i < L; i++)
{
double[] val = null; // this mem. must be inside the loop/allocated for every i and cannot be reused because we need all rows later.
int[] col = null;
int Lr = tis.GetRow(i + i0, ref col, ref val);
data[i] = new Tuple <int, int[], double[]>(Lr, col, val);
}
}
if (Rank > 0)
{
sms.Transmitt(0, data);
}
int rcvProc = 0;
if (Rank == 0)
{
do
{
int i0 = (int)tis.RowPartitioning.GetI0Offest(rcvProc);
if (data.Length != tis.RowPartitioning.GetLocalLength(rcvProc))
{
throw new ApplicationException("internal error");
}
for (int i = 0; i < data.Length; i++)
{
//foreach (MsrMatrix.MatrixEntry entry in data[i]) {
// if (entry.m_ColIndex >= 0)
// ret[i + i0, entry.m_ColIndex] = entry.Value;
//}
int Lr = data[i].Item1;
int[] col = data[i].Item2;
double[] val = data[i].Item3;
for (int lr = 0; lr < Lr; lr++)
{
ret[i + i0, col[lr]] = val[lr];
}
}
} while (sms.GetNext(out rcvProc, out data));
}
else
{
if (sms.GetNext(out rcvProc, out data))
{
throw new ApplicationException("internal error");
}
}
if (Rank == 0)
{
var _ret = MultidimensionalArray.Create(ret.GetLength(0), ret.GetLength(1));
for (int i = 0; i < _ret.NoOfRows; i++)
{
for (int j = 0; j < _ret.NoOfCols; j++)
{
_ret[i, j] = ret[i, j];
}
}
return(_ret);
}
else
{
return(null);
}
}
/// <summary>
/// MPI update of a bit-array
/// </summary>
/// <param name="b"></param>
/// <param name="GridData"></param>
static public void MPIExchange(this BitArray b, IGridData GridData)
{
if (b.Length != GridData.iLogicalCells.NoOfCells)
{
throw new ArgumentException("length must be equal to number of cells.", "b");
}
if (GridData.CellPartitioning.MpiSize > 1)
{
// external
{
int rank, size;
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank);
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out size);
// setup messenger
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
sms.SetCommPathsAndCommit(GridData.iParallel.ProcessesToSendTo);
// send data
for (int p = 0; p < size; p++)
{
int[] sendlist = GridData.iParallel.SendCommLists[p];
if (sendlist == null)
{
continue;
}
int L = sendlist.Length;
System.Collections.BitArray packet_for_p = new System.Collections.BitArray(L, false);
for (int l = 0; l < L; l++)
{
packet_for_p[l] = b[sendlist[l]];
}
sms.Transmitt(p, packet_for_p);
}
// receive data
System.Collections.BitArray rcv_dat;
int rcv_rank;
while (sms.GetNext(out rcv_rank, out rcv_dat))
{
int insertAt = GridData.iParallel.RcvCommListsInsertIndex[rcv_rank];
if (GridData.iParallel.RcvCommListsNoOfItems[rcv_rank] != rcv_dat.Count)
{
throw new ApplicationException("internal error.");
}
int C = rcv_dat.Count;
for (int i = 0; i < C; i++)
{
b[insertAt + i] = rcv_dat[i];
}
}
// dispose
sms.Dispose();
}
}
}
/// <summary>
/// matrix assembly; must be called by each implementation,
/// </summary>
/// <param name="M"></param>
protected void PackMatrix(IMutableMatrixEx M)
{
ilPSP.MPICollectiveWatchDog.Watch();
IPartitioning rp = M.RowPartitioning;
IPartitioning cp = m_ColPart;
// define Comm List
// ================
SortedDictionary <int, List <int> > CommLists = new SortedDictionary <int, List <int> >();
// keys: processor rank p
// values: List of global indices, which processor p needs to send to this processor
int Lr;
double[] val = null;
int[] col = null;
int L = rp.LocalLength;
int i0 = (int)rp.i0;
for (int iLoc = 0; iLoc < L; iLoc++) // loop over all matrix rows...
{
int iGlob = i0 + iLoc;
//MsrMatrix.MatrixEntry[] row = (asMsr==null) ? M.GetRow(iGlob) : asMsr.GetRowShallow(iGlob);
Lr = M.GetOccupiedColumnIndices(iGlob, ref col);
for (int j = 0; j < Lr; j++) // loop over all nonzero entries in row 'iGlob'
{
int jGlob = col[j];
if (cp.i0 <= jGlob && jGlob < (cp.i0 + cp.LocalLength))
{
// Entry on current processor
}
else
{
int proc = cp.FindProcess(jGlob);
// Entry on Processor proc
if (!CommLists.ContainsKey(proc))
{
CommLists.Add(proc, new List <int>());
}
List <int> CommList_proc = CommLists[proc];
if (!CommList_proc.Contains(jGlob)) // a lot of room for optimization
{
CommList_proc.Add(jGlob);
}
}
}
}
// sort com list
// =============
{
foreach (List <int> cl in CommLists.Values)
{
cl.Sort();
}
}
// define matrix
// =============
{
TempCSR intTmp = new TempCSR();
SortedDictionary <int, ExternalTmp> extTmp = new SortedDictionary <int, ExternalTmp>();
foreach (int proc in CommLists.Keys)
{
extTmp.Add(proc, new ExternalTmp());
}
for (int iLoc = 0; iLoc < L; iLoc++)
{
int iGlob = i0 + iLoc;
Lr = M.GetRow(iGlob, ref col, ref val);
for (int j = 0; j < Lr; j++)
{
int jGlob = col[j];
double Value = val[j];
bool bIsDiag = (iGlob == jGlob);
if (cp.i0 <= jGlob && jGlob < (cp.i0 + cp.LocalLength))
{
// Entry on current processor
intTmp.AddEntry(jGlob - (int)cp.i0, Value, bIsDiag);
}
else
{
int proc = cp.FindProcess(jGlob);
// Entry on Processor proc
List <int> CommList_proc = CommLists[proc];
int jloc = CommList_proc.IndexOf(jGlob);
ExternalTmp et = extTmp[proc];
et.AddEntry(jloc, jGlob, Value);
}
}
intTmp.NextRow();
foreach (ExternalTmp et in extTmp.Values)
{
et.NextRow();
}
}
m_LocalMtx = AssembleFinalFormat(intTmp);
ExtMatrix = new Dictionary <int, External>();
foreach (int proc in extTmp.Keys)
{
ExtMatrix.Add(proc, extTmp[proc].GetFinalObj());
}
}
// send/receive & transform Comm lists
// ====================================
{
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
SortedDictionary <int, int[]> CommListsTo = new SortedDictionary <int, int[]>();
foreach (int proc in CommLists.Keys)
{
sms.SetCommPath(proc);
}
sms.CommitCommPaths();
foreach (int proc in CommLists.Keys)
{
sms.Transmitt(proc, CommLists[proc].ToArray());
}
int _proc;
int[] CommListReceived;
sms.GetNext(out _proc, out CommListReceived);
int Lcol = m_ColPart.LocalLength;
int i0col = (int)m_ColPart.i0;
while (CommListReceived != null)
{
// convert indices to local coordinates
for (int i = 0; i < CommListReceived.Length; i++)
{
CommListReceived[i] -= i0col;
// check:
if (CommListReceived[i] < 0 || CommListReceived[i] >= Lcol)
{
throw new ApplicationException("internal error: something wrong with received Comm List.");
}
}
CommListsTo.Add(_proc, CommListReceived);
sms.GetNext(out _proc, out CommListReceived);
}
sms.Dispose();
m_SpmvCommPattern = new SpmvCommPattern();
m_SpmvCommPattern.ComLists = CommListsTo;
}
// record the number of elements which we receive
// ==============================================
{
m_SpmvCommPattern.NoOfReceivedEntries = new Dictionary <int, int>();
foreach (int p in CommLists.Keys)
{
m_SpmvCommPattern.NoOfReceivedEntries.Add(p, CommLists[p].Count);
}
}
}
/// <summary>
/// redistributes the grid, i.e. sends cells to different processors
/// </summary>
/// <param name="part">
/// MPI processor rank for each cell; index: local cell index;
/// </param>
public void RedistributeGrid(int[] part)
{
int Size;
int MyRank;
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out MyRank);
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out Size);
// partition is no longer valid anymore!
m_CellPartitioning = null;
//
int J = NoOfUpdateCells;
if (part.Length < J)
{
throw new ArgumentException();
}
// send cells to other processors
// ==============================
// count number of items for each processor
int[] ItmCnt = new int[Size];
for (int j = 0; j < J; j++)
{
ItmCnt[part[j]]++;
}
// create messenger
SerialisationMessenger sm = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
for (int p = 0; p < Size; p++)
{
if (ItmCnt[p] > 0 && p != MyRank)
{
sm.SetCommPath(p);
}
}
sm.CommitCommPaths();
// sort cells according to processors (part notes which proc. will get a cell)
List <Cell>[] exch = new List <Cell> [Size];
for (int p = 0; p < Size; p++)
{
if (ItmCnt[p] > 0 || p == MyRank)
{
exch[p] = new List <Cell>(ItmCnt[p]);
}
}
for (int j = 0; j < J; j++)
{
exch[part[j]].Add(Cells[j]);
}
// start transmission
for (int p = 0; p < Size; p++)
{
if (exch[p] != null && p != MyRank)
{
sm.Transmitt(p, exch[p]);
}
}
// receive cells from other processors
var myown = exch[MyRank];
List <Cell> o;
int prc;
while (sm.GetNext <List <Cell> >(out prc, out o))
{
myown.AddRange(o);
}
// write back data to arrays
this.Cells = myown.ToArray();
}
/// <summary>
/// Computes a grid partitioning (which cell should be on which processor)
/// using the serial METIS library -- work is only done on MPi rank 0.
/// </summary>
/// <param name="cellWeightsLocal">
/// If not null, defines the weight associted with each cell on the current process
/// </param>
/// <param name="noOfPartitioningsToChooseFrom">
/// Tells METIS to compute
/// </param>
/// <returns>
/// Index: local cell index, content: MPI Processor rank;<br/>
/// This is the suggestion
/// of ParMETIS for the grid partitioning:
/// For each local cell index, the returned array contains the MPI
/// process rank where the cell should be placed.
/// </returns>
public int[] ComputePartitionMETIS(int[] cellWeightsLocal = null, int noOfPartitioningsToChooseFrom = 1)
{
using (new FuncTrace()) {
int size = this.Size;
int rank = this.MyRank;
if (size == 1)
{
return(new int[NoOfUpdateCells]);
}
if (this.NumberOfCells_l > int.MaxValue)
{
throw new Exception(String.Format(
"Grid contains more than {0} cells and can thus not be partitioned using METIS. Use ParMETIS instead.",
int.MaxValue));
}
int J = (rank == 0) ? this.NumberOfCells : 0;
// Setup communication; all send to rank 0
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
if (rank != 0)
{
sms.SetCommPath(0);
}
sms.CommitCommPaths();
// Assemble adjacency lists on rank 0
IEnumerable <Neighbour>[] neighboursGlobal = new IEnumerable <Neighbour> [J];
{
IEnumerable <Neighbour>[] neighboursLocal = GetCellNeighbourship(IncludeBcCells: false).Take(NoOfUpdateCells).ToArray();
if (rank == 0)
{
int localOffset = m_CellPartitioning.GetI0Offest(rank);
int localLength = m_CellPartitioning.GetLocalLength(rank);
for (int i = 0; i < localLength; i++)
{
neighboursGlobal[localOffset + i] = neighboursLocal[i];
}
}
else
{
sms.Transmitt(0, neighboursLocal);
}
while (sms.GetNext(out int senderRank, out IEnumerable <Neighbour>[] neighbours))
{
int localOffset = m_CellPartitioning.GetI0Offest(senderRank);
int localLength = m_CellPartitioning.GetLocalLength(senderRank);
if (neighbours.Length != localLength)
{
throw new Exception();
}
for (int i = 0; i < localLength; i++)
{
neighboursGlobal[localOffset + i] = neighbours[i];
}
}
}
// Gather global weights on rank 0
int[] cellWeightsGlobal = null;
if (cellWeightsLocal != null)
{
cellWeightsGlobal = new int[J];
if (rank == 0)
{
int localOffset = m_CellPartitioning.GetI0Offest(rank);
int localLength = m_CellPartitioning.GetLocalLength(rank);
for (int i = 0; i < localLength; i++)
{
cellWeightsGlobal[localOffset + i] = cellWeightsLocal[i];
}
}
else
{
sms.Transmitt(0, cellWeightsLocal);
}
while (sms.GetNext(out int senderRank, out int[] cellWeights))
{
int localOffset = m_CellPartitioning.GetI0Offest(senderRank);
int localLength = m_CellPartitioning.GetLocalLength(senderRank);
if (cellWeights.Length != localLength)
{
throw new Exception();
}
for (int i = 0; i < localLength; i++)
{
cellWeightsGlobal[localOffset + i] = cellWeights[i];
}
}
}
int[] globalResult = new int[J];
if (rank == 0)
{
int[] xadj = new int[J + 1];
List <int> adjncy = new List <int>(J * m_RefElements[0].NoOfFaces);
for (int j = 0; j < J; j++)
{
var cNj = neighboursGlobal[j];
int E = cNj.Count();
for (int e = 0; e < E; e++)
{
var NN = cNj.ElementAt(e);
if (NN.Neighbour_GlobalIndex >= 0 &&
!NN.IsPeriodicNeighbour)
{
adjncy.Add((int)NN.Neighbour_GlobalIndex);
}
}
xadj[j + 1] = adjncy.Count;
}
// Call METIS
int nparts = size;
Debug.Assert((cellWeightsGlobal == null) == (cellWeightsLocal == null));
int ncon = 1; // One weight per vertex/cell
int objval = -1; // Value of the objective function at return time
int[] Options = new int[METIS.METIS_NOPTIONS];
Options[(int)METIS.OptionCodes.METIS_OPTION_NCUTS] = noOfPartitioningsToChooseFrom; // 5 cuts
Options[(int)METIS.OptionCodes.METIS_OPTION_NITER] = 10; // This is the default refinement iterations
Options[(int)METIS.OptionCodes.METIS_OPTION_UFACTOR] = 30; // Maximum imbalance of 3 percent (this is the default kway clustering)
METIS.ReturnCodes status = (METIS.ReturnCodes)METIS.PartGraphKway(
nvtxs: ref J,
ncon: ref ncon,
xadj: xadj,
adjncy: adjncy.ToArray(),
vwgt: cellWeightsGlobal, // if null, METIS assumes all have weight 1
vsize: null, // No information about communication size
adjwgt: null, // No edge weights
nparts: ref nparts,
tpwgts: null, // No weights for partition constraints
ubvec: null, // No imbalance tolerance for constraints
options: Options,
objval: ref objval,
part: globalResult);
if (status != METIS.ReturnCodes.METIS_OK)
{
throw new Exception(String.Format(
"Error partitioning the mesh. METIS reported {0}",
status));
}
int[] CountCheck = new int[size];
int J2 = this.NumberOfCells;
for (int i = 0; i < J2; i++)
{
CountCheck[globalResult[i]]++;
}
for (int rnk = 0; rnk < size; rnk++)
{
if (CountCheck[rnk] <= 0)
{
throw new ApplicationException("METIS produced illegal partitioning - 0 cells on process " + rnk + ".");
}
}
}
int[] localLengths = new int[size];
for (int p = 0; p < localLengths.Length; p++)
{
localLengths[p] = this.CellPartitioning.GetLocalLength(p);
}
int[] localResult = globalResult.MPIScatterv(localLengths);
return(localResult);
}
}
/// <summary>
/// detects how the MPI nodes are distributed over compute nodes (SMP nodes)
/// </summary>
private void SMPEvaluation()
{
//int ht = m_Context.IOMaster.tracer.EnterFunction("BoSSS.Foundation.Comm.DatabaseDriver.SMPEvaluation");
using (new FuncTrace()) {
ilPSP.MPICollectiveWatchDog.Watch(MPI.Wrappers.csMPI.Raw._COMM.WORLD);
// define SMP rank;
// for each MPI process, the SMP node index
// index: MPI rank; content: SMP rank;
int[] SMPRank = null;
int NoOfSMPs = -1;
{
// we are using the computer name to determine
// which MPI processes run on the same physical machine
// send host name to proc 0.
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
if (MyRank > 0)
{
sms.SetCommPath(0);
}
sms.CommitCommPaths();
if (MyRank > 0)
{
sms.Transmitt(0, m_hostname);
}
int recvRnk;
string nmn;
sms.GetNext(out recvRnk, out nmn);
if (MyRank == 0)
{
// receiving names form all processors
List <string> hosts_unique = new List <string>();
hosts_unique.Add(m_hostname);
string[] hosts = new string[Size];
SMPRank = new int[Size];
ArrayTools.SetAll(SMPRank, int.MinValue);
hosts[0] = m_hostname;
SMPRank[0] = hosts_unique.IndexOf(m_hostname);
while (nmn != null)
{
if (hosts[recvRnk] != null)
{
throw new ApplicationException("should not happen.");
}
hosts[recvRnk] = nmn;
int smpRnk = hosts_unique.IndexOf(nmn);
if (smpRnk < 0)
{
hosts_unique.Add(nmn);
smpRnk = hosts_unique.Count - 1;
}
SMPRank[recvRnk] = smpRnk;
sms.GetNext(out recvRnk, out nmn);
}
NoOfSMPs = hosts_unique.Count;
for (int i = 0; i < Size; i++)
{
if (hosts[i] == null || SMPRank[i] < 0)
{
throw new ApplicationException("fatal error in algorithm.");
}
}
}
else
{
// don't receive anything
if (nmn != null)
{
// fatal error in algorithm
throw new ApplicationException("ha?");
}
}
sms.Dispose();
}
m_SMPSize = NoOfSMPs.MPIBroadcast(0, csMPI.Raw._COMM.WORLD);
m_SMPRanks = SMPRank.MPIBroadcast(0, csMPI.Raw._COMM.WORLD);
{
// number of MPI processes per SMP rank; index: SMP rank
m_MPIProcessesPerSMP = new int[m_SMPSize];
int[] _MPIProcessesPerSMP = new int[m_SMPSize];
_MPIProcessesPerSMP[m_SMPRanks[m_MyRank]]++;
unsafe
{
fixed(int *pSnd = &_MPIProcessesPerSMP[0], pRcv = &m_MPIProcessesPerSMP[0])
{
csMPI.Raw.Allreduce((IntPtr)pSnd, (IntPtr)pRcv, m_SMPSize, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.SUM, csMPI.Raw._COMM.WORLD);
}
}
}
//m_Context.IOMaster.tracer.LeaveFunction(ht);
}
}
/// <summary>
/// ctor.
/// </summary>
/// <param name="M"></param>
/// <param name="ExtCol">
/// key: processor rank 'p' <br/>
/// value: a list of column indices (within the local range of columns of <paramref name="M"/>),
/// which should be editable at rank 'p'.
/// </param>
public MsrExtMatrix(IMutableMatrixEx M, IDictionary <int, int[]> ExtCol)
{
this.ColPart = M.ColPartition;
int i0Row = (int)M.RowPartitioning.i0, I = M.RowPartitioning.LocalLength,
i0Col = (int)this.ColPart.i0, J = this.ColPart.LocalLength;
Mtx = M;
// init
// ====
ColToRowLocal = new List <int> [ColPart.LocalLength];
for (int j = 0; j < ColToRowLocal.Length; j++)
{
ColToRowLocal[j] = new List <int>();
}
// build Column to row - mapping
// =============================
ColToRowExternal = new Dictionary <int, List <int> >();
// key: global column index j, within the range of processor 'p'
// values: global row indices
SortedDictionary <int, List <int> > ColForProc = new SortedDictionary <int, List <int> >();
// key: MPI processor index 'p'
// values: a set of global column indices, within the range of processor 'p',
// that contain nonzero entries on this processor
int[] col = null;
int L;
// loop over all rows...
for (int i = 0; i < I; i++)
{
L = M.GetOccupiedColumnIndices(i + i0Row, ref col);
// loop over all nonzero entries in the row...
for (int l = 0; l < L; l++)
{
int ColIndex = col[l];
int localColInd = ColIndex - i0Col;
if (localColInd >= 0 && localColInd < J)
{
// column of 'entry' is within the local range of this processor
// + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColToRowLocal[localColInd].Add(i + i0Row);
}
else
{
// column of 'entry' belongs to external processor 'proc'
// + + + + + + + + + + + + + + + + + + + + + + + + + + + +
int proc = this.ColPart.FindProcess(ColIndex);
{
//SortedDictionary<int, List<int>> ColToRowExt_proc;
//if (!ColToRowExternal.ContainsKey(proc)) {
// ColToRowExt_proc = new SortedDictionary<int, List<int>>();
// ColToRowExternal.Add(proc, ColToRowExt_proc);
//} else {
// ColToRowExt_proc = ColToRowExternal[proc];
//}
int j = ColIndex;
List <int> Rows4Col;
if (!ColToRowExternal.ContainsKey(j))
{
Rows4Col = new List <int>();
ColToRowExternal.Add(j, Rows4Col);
}
else
{
Rows4Col = ColToRowExternal[j];
}
Rows4Col.Add(i + i0Row);
}
{
List <int> ColForProc_proc;
if (!ColForProc.ContainsKey(proc))
{
ColForProc_proc = new List <int>();
ColForProc.Add(proc, ColForProc_proc);
}
else
{
ColForProc_proc = ColForProc[proc];
}
if (!ColForProc_proc.Contains(ColIndex))
{
ColForProc_proc.Add(ColIndex);
}
}
}
}
}
// communicate
// ===========
//SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw.MPI_COMM_WORLD);
//{
// foreach (int proc in ColToRowExternal.Keys) {
// sms.SetCommPath(proc);
// }
// sms.CommitCommPaths();
// // send
// foreach (int proc in ColToRowExternal.Keys) {
// SortedDictionary<int, List<int>> ColToRowExt_proc = ColToRowExternal[proc];
// SortedList _ColToRowExt_proc = new SortedList();
// foreach (int iCol in ColToRowExt_proc.Keys)
// _ColToRowExt_proc.Add(iCol, ColToRowExt_proc[iCol].ToArray());
// }
// // receive
// int p; SortedList rcv;
// sms.GetNext(out p, out rcv);
// while (rcv != null) {
// foreach (int col in rcv.Keys) {
// int[] rowList = (int[])rcv[col];
// ColToRowLocal[col].AddRange(rowList);
// }
// sms.GetNext(out p, out rcv);
// }
//}
//sms.Dispose();
// build 'ColProcessors'
// =====================
//ColProcessors = new List<int>[ColToRowLocal.Length];
//for (int j = 0; j < ColToRowLocal.Length; j++) {
// List<int> mpiRank = null;
// foreach (int rowind in ColToRowLocal[j]) {
// int riloc = rowind - i0Row;
// if (riloc < 0 || riloc >= I) {
// if (mpiRank == null)
// mpiRank = new List<int>();
// mpiRank.Add(M.RowPartiton.FindProcess(rowind));
// }
// ColProcessors[j] = mpiRank;
// }
//}
// communicate: build 'ColProcessors'
// ==================================
{
ColProcessors = new List <int> [ColPart.LocalLength];
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
sms.SetCommPathsAndCommit(ColForProc.Keys);
foreach (int proc in ColForProc.Keys)
{
sms.Transmitt(proc, ColForProc[proc].ToArray());
}
int i0Loc = (int)ColPart.i0;
int rcvproc; int[] ColIndices;
while (sms.GetNext(out rcvproc, out ColIndices))
{
int Rank;
{
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out Rank);
//Console.WriteLine("P# " + Rank + ": receiving from P# " + rcvproc);
}
foreach (int ColInd in ColIndices)
{
int localColInd = ColInd - i0Loc;
if (localColInd < 0 || localColInd >= ColPart.LocalLength)
{
throw new IndexOutOfRangeException("internal error");
}
if (ColProcessors[localColInd] == null)
{
ColProcessors[localColInd] = new List <int>();
}
if (ColProcessors[localColInd].Contains(rcvproc))
{
throw new ApplicationException("internal error.");
}
ColProcessors[localColInd].Add(rcvproc);
}
}
sms.Dispose();
}
if (ExtCol != null)
{
var send = new Dictionary <int, List <Tuple <int, List <int> > > >();
int myRank = M.RowPartitioning.MpiRank;
foreach (var kv in ExtCol)
{
int rank = kv.Key;
int[] ColIdx = kv.Value;
var sendToRank = new List <Tuple <int, List <int> > >();
foreach (int iCol in ColIdx)
{
List <int> c2p = ColProcessors[iCol - i0Col];
var t = new Tuple <int, List <int> >(iCol, c2p != null ? new List <int>(c2p) : new List <int>());
t.Item2.Add(myRank);
sendToRank.Add(t);
}
send.Add(rank, sendToRank);
}
var receive = SerialisationMessenger.ExchangeData(send, csMPI.Raw._COMM.WORLD);
ColProcessorsExternal = new Dictionary <int, List <int> >();
foreach (var kv in receive)
{
var val = kv.Value;
foreach (var t in val)
{
int iCol = t.Item1;
List <int> ranks = t.Item2;
int iMyRank = ranks.IndexOf(myRank);
if (iMyRank >= 0)
{
ranks.RemoveAt(iMyRank);
}
ColProcessorsExternal.Add(t.Item1, t.Item2);
Debug.Assert(this.ColPart.FindProcess(t.Item1) == kv.Key);
}
}
#if DEBUG
foreach (var procList in ColProcessorsExternal.Values)
{
Debug.Assert(procList.Contains(myRank) == false);
}
#endif
}
}
/// <summary>
/// An MPI-collective call, which executes all column operations.
/// </summary>
public void CompleteColOperation()
{
int j0Loc = (int)m_Matrix.ColPart.i0;
int LenLoc = m_Matrix.ColPart.LocalLength;
// sort operations according to processor
// ======================================
// keys: MPI processor rank p
// values: list of operations to execute on p
SortedDictionary <int, List <ColOp> > OperationsPerProcessor = new SortedDictionary <int, List <ColOp> >();
List <int> InvokedProc;
for (int i = 0; i < DeferredColOpList.Count; i++)
{
ColOp op = DeferredColOpList[i];
bool skip = false;
ColAddition ca = op as ColAddition;
List <int> InvokesProcSrc = null;
if (ca != null)
{
// we have a column addition - this requires some special treatments
// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// problem 1: if
// * the destination col. (aka. accumulator column), i.e. column no. 'op.jCol'
// is zero,
// * and the source row is nonzero
//
// then we need to send the command not only to processors which contain
// nonzeros in destination row, but also the source row.
//
// problem 2: for subsequent operations, we may expect that some column which has
// originally been zero now contains nonzero elements.
// So, therefore, we have to add the processor set of the source row
// (i.e. 'm_Matrix.ColProcessors[ca.iSrc]' to the processor set of the
// destination row.
if (m_Matrix.ColPart.IsInLocalRange(ca.iSrc))
{
InvokesProcSrc = m_Matrix.ColProcessors[ca.iSrc - j0Loc];
}
else
{
if (!this.m_Matrix.ColProcessorsExternal.TryGetValue(ca.iSrc, out InvokesProcSrc))
{
throw new IndexOutOfRangeException("manipulation operation not available on current processor");
}
}
if (InvokesProcSrc != null)
{
if (m_Matrix.ColProcessors[op.jCol - j0Loc] == null)
{
m_Matrix.ColProcessors[op.jCol - j0Loc] = InvokesProcSrc;
}
else
{
InvokedProc = m_Matrix.ColProcessors[op.jCol - j0Loc];
foreach (int ps in InvokesProcSrc)
{
if (!InvokedProc.Contains(ps))
{
InvokedProc.Add(ps);
}
}
}
}
else
{
// optimization: source column is zero (on other processors) -> nothing to do
skip = true;
}
}
if (m_Matrix.ColPart.IsInLocalRange(op.jCol))
{
InvokedProc = m_Matrix.ColProcessors[op.jCol - j0Loc];
}
else
{
if (!this.m_Matrix.ColProcessorsExternal.TryGetValue(op.jCol, out InvokedProc))
{
throw new IndexOutOfRangeException("manipulation operation not available on current processor");
}
}
if (InvokedProc != null && !skip)
{
foreach (int proc in InvokedProc)
{
bool skip2 = false;
if (ca != null)
{
// optimization: don't need to send column addition if
// the source row is zero
if (!InvokesProcSrc.Contains(proc))
{
skip2 = true;
}
}
if (!skip2)
{
List <ColOp> DeferredOp_proc;
if (OperationsPerProcessor.ContainsKey(proc))
{
DeferredOp_proc = OperationsPerProcessor[proc];
}
else
{
DeferredOp_proc = new List <ColOp>();
OperationsPerProcessor.Add(proc, DeferredOp_proc);
}
DeferredOp_proc.Add(op);
}
}
}
}
// transmit to other processors
// ============================
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
foreach (int proc in OperationsPerProcessor.Keys)
{
sms.SetCommPath(proc);
}
sms.CommitCommPaths();
foreach (int proc in OperationsPerProcessor.Keys)
{
sms.Transmitt(proc, OperationsPerProcessor[proc].ToArray());
}
int rcvp; ColOp[] rcv;
while (sms.GetNext(out rcvp, out rcv))
{
DeferredColOpList.AddRange(rcv); // operations from different processors
// commute (because they are bound to the column partition)
// therefore, it doesn't matter how they are added
//#if DEBUG
// {
// int Rank;
// csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out Rank);
// //Console.WriteLine("P# " + Rank + ": " + rcv.Length + " operation(s) received from P# " + rcvp);
// foreach (var op in rcv) {
// if ((op.jCol >= m_Matrix.ColPart.i0 && op.jCol < (m_Matrix.ColPart.i0 + m_Matrix.ColPart.LocalLength)))
// throw new ApplicationException("internal error");
// }
// }
//#endif
}
// execute operations
// ==================
int L = DeferredColOpList.Count;
for (int l = 0; l < L; l++)
{
ColOp op = DeferredColOpList[l];
int jlocal = op.jCol - j0Loc;
if (op is ColAddition)
{
double alpha = ((ColAddition)op).alpha;
int iSrc = ((ColAddition)op).iSrc;
int[] col;
if (iSrc >= j0Loc && iSrc < (j0Loc + LenLoc))
{
// operation in local column range
col = m_Matrix.ColToRowLocal[iSrc - j0Loc].ToArray();
}
else
{
// operation comes from other processor
List <int> _col;
if (m_Matrix.ColToRowExternal.TryGetValue(iSrc, out _col))
{
col = _col.ToArray();
}
else
{
col = new int[0];
}
//col = m_Matrix.ColToRowExternal[iSrc].ToArray();
}
foreach (int irow in col)
{
m_Matrix[irow, op.jCol] += m_Matrix[irow, iSrc] * alpha;
}
}
else
{
int[] col;
if (jlocal >= 0 && jlocal < LenLoc)
{
// operation in local column range
col = m_Matrix.ColToRowLocal[jlocal].ToArray();
}
else
{
// operation comes from other processor
List <int> _col;
if (m_Matrix.ColToRowExternal.TryGetValue(op.jCol, out _col))
{
col = _col.ToArray();
}
else
{
col = new int[0];
}
//col = m_Matrix.ColToRowExternal[op.jCol].ToArray();
}
if (op is ColMul)
{
double alpha = ((ColMul)op).alpha;
foreach (int irow in col)
{
m_Matrix[irow, op.jCol] *= alpha;
}
}
else if (op is ColClear)
{
foreach (int irow in col)
{
m_Matrix[irow, op.jCol] = 0;
}
}
else
{
throw new NotImplementedException();
}
}
}
// finish & return
// ===============
DeferredColOpList.Clear();
}
