public int GetI0Offest(int proc)
{
return(m_Partitioning.GetI0Offest(proc));
}
/// <summary>
/// initializes this matrix as a copy of the matrix <paramref name="M"/>.
/// </summary>
public Matrix(IMutableMatrixEx M, bool UseDoublePrecision)
{
if (M.RowPartitioning.IsMutable)
{
throw new NotSupportedException();
}
if (M.ColPartition.IsMutable)
{
throw new NotSupportedException();
}
if (M.NoOfCols != M.NoOfRows)
{
throw new ArgumentException("Matrix must be quadratic.", "M");
}
this.Symmetric = (M is MsrMatrix) && ((MsrMatrix)M).AssumeSymmetric;
RowPart = M.RowPartitioning;
int size = M.RowPartitioning.MpiSize, rank = M.RowPartitioning.MpiRank;
Debug.Assert(M.RowPartitioning.MpiRank == M.ColPartition.MpiRank);
Debug.Assert(M.RowPartitioning.MpiSize == M.ColPartition.MpiSize);
m_comm = M.MPI_Comm;
int LR;
int[] col = null;
double[] val = null;
if (size == 1)
{
// serial init on one processor
// ++++++++++++++++++++++++++++
n = (int)M.RowPartitioning.TotalLength;
int len;
if (Symmetric)
{
// upper triangle + diagonal (diagonal entries are
// always required, even if 0.0, for symmetric matrices in PARDISO)
len = M.GetGlobalNoOfUpperTriangularNonZeros() + n;
}
else
{
len = M.GetTotalNoOfNonZerosPerProcess();
}
int Nrows = M.RowPartitioning.LocalLength;
int cnt = 0;
ia = new int[n + 1];
ja = new int[len];
if (UseDoublePrecision)
{
a_D = new double[len];
}
else
{
a_S = new float[len];
}
for (int i = 0; i < Nrows; i++)
{
ia[i] = cnt + 1; // fortran indexing
int iRow = M.RowPartitioning.i0 + i;
LR = M.GetRow(iRow, ref col, ref val);
double diagelem = M[iRow, iRow];
if (Symmetric && diagelem == 0)
{
// in the symmetric case, we always need to provide the diagonal element
ja[cnt] = iRow + 1; // fortran indexing
if (UseDoublePrecision)
{
a_D[cnt] = 0.0;
}
else
{
a_S[cnt] = 0.0f;
}
cnt++;
}
for (int j = 0; j < LR; j++)
{
if (val[j] != 0.0)
{
if (Symmetric && col[j] < iRow)
{
// entry is in lower triangular matrix -> ignore (for symmetric mtx.)
continue;
}
else
{
ja[cnt] = col[j] + 1; // fortran indexing
if (UseDoublePrecision)
{
a_D[cnt] = val[j];
}
else
{
a_S[cnt] = (float)(val[j]);
}
cnt++;
}
}
}
//if (M.GetTotalNoOfNonZeros() != len)
// throw new Exception();
}
ia[Nrows] = cnt + 1; // fortran indexing
if (len != cnt)
{
throw new ApplicationException("internal error.");
}
}
else
{
// collect matrix on processor 0
// +++++++++++++++++++++++++++++
// Number of elements, start indices for index pointers
// ====================================================
int len_loc;
if (Symmetric)
{
// number of entries is:
// upper triangle + diagonal (diagonal entries are
// always required, even if 0.0, for symmetric matrices in PARDISO)
len_loc = M.GetLocalNoOfUpperTriangularNonZeros() + M.RowPartitioning.LocalLength;
}
else
{
len_loc = M.GetTotalNoOfNonZerosPerProcess();
}
Partitioning part = new Partitioning(len_loc, m_comm);
if (part.TotalLength > int.MaxValue)
{
throw new ApplicationException("too many matrix entries for PARDISO - more than maximum 32-bit signed integer");
}
// local matrix assembly
// =====================
int n_loc = M.RowPartitioning.LocalLength;
int[] ia_loc = new int[n_loc];
int[] ja_loc = new int[len_loc];
double[] a_loc_D = null;
float[] a_loc_S = null;
if (UseDoublePrecision)
{
a_loc_D = new double[len_loc];
}
else
{
a_loc_S = new float[len_loc];
}
{
int cnt = 0;
int i0 = (int)part.i0;
for (int i = 0; i < n_loc; i++)
{
ia_loc[i] = cnt + 1 + i0; // fortran indexing
int iRow = i + (int)M.RowPartitioning.i0;
LR = M.GetRow(iRow, ref col, ref val);
double diagelem = M[iRow, iRow];
if (Symmetric && diagelem == 0)
{
// in the symmetric case, we always need to provide the diagonal element
ja_loc[cnt] = iRow + 1; // fortran indexing
if (UseDoublePrecision)
{
a_loc_D[cnt] = 0.0;
}
else
{
a_loc_S[cnt] = 0.0f;
}
cnt++;
}
for (int j = 0; j < LR; j++)
{
if (val[j] != 0.0)
{
if (Symmetric && col[j] < iRow)
{
// entry is in lower triangular matrix -> ignore (for symmetric mtx.)
continue;
}
else
{
ja_loc[cnt] = col[j] + 1; // fortran indexing
if (UseDoublePrecision)
{
a_loc_D[cnt] = val[j];
}
else
{
a_loc_S[cnt] = (float)val[j];
}
cnt++;
}
}
}
}
if (cnt != len_loc)
{
throw new ApplicationException("internal error.");
}
}
// assemble complete matrix on proc. 0
// ===================================
if (rank == 0)
{
n = M.RowPartitioning.TotalLength;
// process 0: collect data from other processors
// +++++++++++++++++++++++++++++++++++++++++++++
this.ia = new int[M.RowPartitioning.TotalLength + 1];
this.ja = new int[part.TotalLength];
if (UseDoublePrecision)
{
this.a_D = new double[part.TotalLength];
}
else
{
this.a_S = new float[part.TotalLength];
}
}
unsafe
{
unsafe
{
int *displs = stackalloc int[size];
int *recvcounts = stackalloc int[size];
for (int i = 0; i < size; i++)
{
recvcounts[i] = part.GetLocalLength(i);
displs[i] = part.GetI0Offest(i);
}
fixed(void *pa_loc_D = a_loc_D, pa_D = a_D, pa_loc_S = a_loc_S, pa_S = a_S)
{
if (UseDoublePrecision)
{
csMPI.Raw.Gatherv(
(IntPtr)pa_loc_D,
a_loc_D.Length,
csMPI.Raw._DATATYPE.DOUBLE,
(IntPtr)pa_D,
(IntPtr)recvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.DOUBLE,
0,
m_comm);
}
else
{
csMPI.Raw.Gatherv(
(IntPtr)pa_loc_S,
a_loc_S.Length,
csMPI.Raw._DATATYPE.FLOAT,
(IntPtr)pa_S,
(IntPtr)recvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.FLOAT,
0,
m_comm);
}
}
fixed(void *pja_loc = ja_loc, pja = ja)
{
csMPI.Raw.Gatherv(
(IntPtr)pja_loc,
ja_loc.Length,
csMPI.Raw._DATATYPE.INT,
(IntPtr)pja,
(IntPtr)recvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.INT,
0,
m_comm);
}
for (int i = 0; i < size; i++)
{
displs[i] = M.RowPartitioning.GetI0Offest(i);
recvcounts[i] = M.RowPartitioning.GetLocalLength(i);
}
fixed(void *pia_loc = ia_loc, pia = ia)
{
csMPI.Raw.Gatherv(
(IntPtr)pia_loc,
ia_loc.Length,
csMPI.Raw._DATATYPE.INT,
(IntPtr)pia,
(IntPtr)recvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.INT,
0,
m_comm);
}
}
}
if (rank == 0)
{
this.ia[M.RowPartitioning.TotalLength] = (int)part.TotalLength + 1;
}
ia_loc = null; ja_loc = null; a_loc_S = null; a_loc_D = null;
GC.Collect();
}
}
/// <summary>
/// parallel evaluator; For arguments which are not stored on this process,
/// other processes are asked.
/// </summary>
/// <param name="Keys">input; indices at which the permutation should be evaluated;</param>
/// <param name="Result">
/// output; the i-th entry is the value of the permutation for index
/// <paramref name="Keys"/>[i];
/// </param>
public void EvaluatePermutation <T1, T2>(T1 Keys, T2 Result)
where T1 : IList <long>
where T2 : IList <long> //
{
using (new FuncTrace()) {
if (Keys.Count != Result.Count)
{
throw new ArgumentException("Keys and Result array must have the same number of elements");
}
int cnt = Keys.Count;
int myrank;
int size;
MPI.Wrappers.csMPI.Raw.Comm_Size(this.m_Comm, out size);
MPI.Wrappers.csMPI.Raw.Comm_Rank(this.m_Comm, out myrank);
ilPSP.MPICollectiveWatchDog.Watch(this.m_Comm);
// create objects for questions to other processors
// ================================================
List <int>[] Questions = new List <int> [size];
List <int>[] iQuestions = new List <int> [size];
for (int p = 0; p < size; p++)
{
Questions[p] = new List <int>();
iQuestions[p] = new List <int>();
}
// local evaluation of the permutation mapping
// ===========================================
for (int i = 0; i < cnt; i++)
{
long idx = Keys[i];
if (idx < 0 || idx >= TotalLength)
{
throw new ArgumentException("argument out of range; (argument values start at 0)");
}
long idxLocal = idx - i0Offset;
if (idxLocal >= 0 && idxLocal < LocalLength)
{
// local evaluation
Result[i] = m_Values[idxLocal]; // that's all, folks
}
else
{
// another processor has to be asked
int targProcess = m_Partition.FindProcess(idx);
int idxLoc = (int)(idx - m_Partition.GetI0Offest(targProcess)); // m_i0Offset[targProcess]);
Questions[targProcess].Add(idxLoc);
iQuestions[targProcess].Add(i);
}
}
// --------------------
// Ask other processors
// --------------------
// setup question messenger
// ========================
Many2ManyMessenger <int> m2mAsk = new Many2ManyMessenger <int>(m_Comm);
for (int p = 0; p < size; p++)
{
if (Questions[p].Count > 0)
{
m2mAsk.SetCommPath(p, Questions[p].Count);
}
}
m2mAsk.CommitCommPaths();
// transmit data
// =============
m2mAsk.StartTransmission(1);
for (int p = 0; p < size; p++)
{
if (Questions[p].Count > 0)
{
m2mAsk.SendBuffers(p).CopyFrom(Questions[p].ToArray(), 0);
Questions[p] = null;
m2mAsk.TransmittData(p);
}
}
// wait until communication is finished
// ====================================
m2mAsk.FinishBlocking();
// --------------------------
// answer to other processors
// --------------------------
// setup answer messenger
// ======================
Many2ManyMessenger <long> m2mAnswer = new Many2ManyMessenger <long>(m_Comm);
for (int p = 0; p < size; p++)
{
if (m2mAsk.ReceiveBuffers(p) != null)
{
m2mAnswer.SetCommPath(p, m2mAsk.ReceiveBuffers(p).Count);
}
}
m2mAnswer.CommitCommPaths();
// transmit data
// =============
m2mAnswer.StartTransmission(1);
for (int p = 0; p < size; p++)
{
Many2ManyMessenger <int> .Buffer q = m2mAsk.ReceiveBuffers(p);
Many2ManyMessenger <long> .Buffer r = m2mAnswer.SendBuffers(p);
if (q != null)
{
int _cnt = q.Count;
for (int i = 0; i < _cnt; i++)
{
r[i] = m_Values[q[i]];
}
m2mAnswer.TransmittData(p);
}
}
m2mAsk.Dispose();
// wait until communication is finished
// ====================================
m2mAnswer.FinishBlocking();
// ------------------------------------------
// evaluate the answers from other processors
// ------------------------------------------
for (int p = 0; p < size; p++)
{
Many2ManyMessenger <long> .Buffer r = m2mAnswer.ReceiveBuffers(p);
if (r != null)
{
// test
if (r.Count != iQuestions[p].Count)
{
throw new ApplicationException("internal error: mismatch between number of questions/answers");
}
int c = iQuestions[p].Count;
List <int> iQeus = iQuestions[p];
for (int i = 0; i < c; i++)
{
Result[iQeus[i]] = r[i];
}
}
}
m2mAnswer.Dispose();
}
}
