/// <summary>
/// constructor;
/// Cannot be called before communication paths of <paramref name="owner"/> are committed
/// (<see cref="Many2ManyMessenger{ItemType}.CommitCommPaths"/>);
/// </summary>
/// <param name="owner"></param>
/// <param name="ReceiveOrTransmit">
/// true: a receive buffer
/// false: a transmit (or send) buffer
/// </param>
/// <param name="ProcRank"></param>
internal Buffer(Many2ManyMessenger <ItemType> owner, bool ReceiveOrTransmit, int ProcRank)
{
if (owner.m_ItemType != typeof(ItemType))
{
throw new ArgumentException("Item type varies from messenger item type");
}
m_Owner = owner;
m_ProcRank = ProcRank;
m_ReceiveOrTransmit = ReceiveOrTransmit;
if (!m_Owner.m_CommPathsCommited)
{
throw new ApplicationException("communication paths must be commited first.");
}
}
/// <summary>
/// parallel evaluator; For arguments which are not stored on this process,
/// other processes are asked.
/// </summary>
/// <param name="Keys">input; indices at which the permutation should be evaluated;</param>
/// <param name="Result">
/// output; the i-th entry is the value of the permutation for index
/// <paramref name="Keys"/>[i];
/// </param>
public void EvaluatePermutation <T1, T2>(T1 Keys, T2 Result)
where T1 : IList <long>
where T2 : IList <long> //
{
using (new FuncTrace()) {
if (Keys.Count != Result.Count)
{
throw new ArgumentException("Keys and Result array must have the same number of elements");
}
int cnt = Keys.Count;
int myrank;
int size;
MPI.Wrappers.csMPI.Raw.Comm_Size(this.m_Comm, out size);
MPI.Wrappers.csMPI.Raw.Comm_Rank(this.m_Comm, out myrank);
ilPSP.MPICollectiveWatchDog.Watch(this.m_Comm);
// create objects for questions to other processors
// ================================================
List <int>[] Questions = new List <int> [size];
List <int>[] iQuestions = new List <int> [size];
for (int p = 0; p < size; p++)
{
Questions[p] = new List <int>();
iQuestions[p] = new List <int>();
}
// local evaluation of the permutation mapping
// ===========================================
for (int i = 0; i < cnt; i++)
{
long idx = Keys[i];
if (idx < 0 || idx >= TotalLength)
{
throw new ArgumentException("argument out of range; (argument values start at 0)");
}
long idxLocal = idx - i0Offset;
if (idxLocal >= 0 && idxLocal < LocalLength)
{
// local evaluation
Result[i] = m_Values[idxLocal]; // that's all, folks
}
else
{
// another processor has to be asked
int targProcess = m_Partition.FindProcess(idx);
int idxLoc = (int)(idx - m_Partition.GetI0Offest(targProcess)); // m_i0Offset[targProcess]);
Questions[targProcess].Add(idxLoc);
iQuestions[targProcess].Add(i);
}
}
// --------------------
// Ask other processors
// --------------------
// setup question messenger
// ========================
Many2ManyMessenger <int> m2mAsk = new Many2ManyMessenger <int>(m_Comm);
for (int p = 0; p < size; p++)
{
if (Questions[p].Count > 0)
{
m2mAsk.SetCommPath(p, Questions[p].Count);
}
}
m2mAsk.CommitCommPaths();
// transmit data
// =============
m2mAsk.StartTransmission(1);
for (int p = 0; p < size; p++)
{
if (Questions[p].Count > 0)
{
m2mAsk.SendBuffers(p).CopyFrom(Questions[p].ToArray(), 0);
Questions[p] = null;
m2mAsk.TransmittData(p);
}
}
// wait until communication is finished
// ====================================
m2mAsk.FinishBlocking();
// --------------------------
// answer to other processors
// --------------------------
// setup answer messenger
// ======================
Many2ManyMessenger <long> m2mAnswer = new Many2ManyMessenger <long>(m_Comm);
for (int p = 0; p < size; p++)
{
if (m2mAsk.ReceiveBuffers(p) != null)
{
m2mAnswer.SetCommPath(p, m2mAsk.ReceiveBuffers(p).Count);
}
}
m2mAnswer.CommitCommPaths();
// transmit data
// =============
m2mAnswer.StartTransmission(1);
for (int p = 0; p < size; p++)
{
Many2ManyMessenger <int> .Buffer q = m2mAsk.ReceiveBuffers(p);
Many2ManyMessenger <long> .Buffer r = m2mAnswer.SendBuffers(p);
if (q != null)
{
int _cnt = q.Count;
for (int i = 0; i < _cnt; i++)
{
r[i] = m_Values[q[i]];
}
m2mAnswer.TransmittData(p);
}
}
m2mAsk.Dispose();
// wait until communication is finished
// ====================================
m2mAnswer.FinishBlocking();
// ------------------------------------------
// evaluate the answers from other processors
// ------------------------------------------
for (int p = 0; p < size; p++)
{
Many2ManyMessenger <long> .Buffer r = m2mAnswer.ReceiveBuffers(p);
if (r != null)
{
// test
if (r.Count != iQuestions[p].Count)
{
throw new ApplicationException("internal error: mismatch between number of questions/answers");
}
int c = iQuestions[p].Count;
List <int> iQeus = iQuestions[p];
for (int i = 0; i < c; i++)
{
Result[iQeus[i]] = r[i];
}
}
}
m2mAnswer.Dispose();
}
}
/// <summary>
/// performs a parallel inversion of this permutation
/// </summary>
/// <returns></returns>
public Permutation Invert()
{
using (new FuncTrace()) {
Permutation inv = new Permutation(this);
int localLength = m_Partition.LocalLength; //m_LocalLengths[m_Master.MyRank];
int size;
MPI.Wrappers.csMPI.Raw.Comm_Size(m_Comm, out size);
inv.m_Values = new long[localLength];
long myi0Offset = m_Partition.i0; //m_i0Offset[m_Master.MyRank];
Many2ManyMessenger <PermutationEntry> m2m = new Many2ManyMessenger <PermutationEntry>(m_Comm);
int[] NoOfItemsToSent = new int[size];
// calc i0's for each process
// ==========================
long[] i0 = new long[size + 1];
for (int i = 1; i <= size; i++)
{
i0[i] = i0[i - 1] + m_Partition.GetLocalLength(i - 1);// m_LocalLengths[i - 1];
}
// do local inversion and collect items
// ====================================
List <PermutationEntry>[] itemsToSend = new List <PermutationEntry> [size];
for (int p = 0; p < size; p++)
{
itemsToSend[p] = new List <PermutationEntry>();
}
for (int i = 0; i < localLength; i++)
{
// decide wether inversion of entry i is local or has to be transmitted ...
if (m_Values[i] >= myi0Offset && m_Values[i] < (myi0Offset + localLength))
{
// local
inv.m_Values[this.m_Values[i] - myi0Offset] = i + myi0Offset;
}
else
{
// transmit
// find target processor
int targProc = m_Partition.FindProcess(m_Values[i]);
NoOfItemsToSent[targProc]++;
// collect item
PermutationEntry pe;
pe.Index = myi0Offset + i;
pe.PermVal = m_Values[i];
itemsToSend[targProc].Add(pe);
}
}
// setup messenger
// ===============
for (int i = 0; i < size; i++)
{
if (NoOfItemsToSent[i] > 0)
{
m2m.SetCommPath(i, NoOfItemsToSent[i]);
}
}
m2m.CommitCommPaths();
// transmit data
// =============
//csMPI.Raw.Barrier(csMPI.Raw.MPI_COMM_WORLD);
//Console.WriteLine("one");
//csMPI.Raw.Barrier(csMPI.Raw.MPI_COMM_WORLD);
//Console.WriteLine("two");
//csMPI.Raw.Barrier(csMPI.Raw.MPI_COMM_WORLD);
//Console.WriteLine("three");
m2m.StartTransmission(1);
for (int p = 0; p < size; p++)
{
if (NoOfItemsToSent[p] <= 0)
{
continue;
}
//Many2ManyMessenger.Buffer<PermEntry> sndbuf = new Many2ManyMessenger.Buffer<PermEntry>(m2m, false, p);
PermutationEntry[] items = itemsToSend[p].ToArray();
m2m.SendBuffers(p).CopyFrom(items, 0);
m2m.TransmittData(p);
}
m2m.FinishBlocking();
// invert received items
// =====================
for (int p = 0; p < size; p++)
{
Many2ManyMessenger <PermutationEntry> .Buffer rcvbuf = m2m.ReceiveBuffers(p);
if (rcvbuf == null)
{
continue; // no data from process no. p
}
int cnt = rcvbuf.Count;
PermutationEntry[] items = new PermutationEntry[cnt];
rcvbuf.CopyTo(items, 0);
for (int i = 0; i < cnt; i++)
{
inv.m_Values[items[i].PermVal - myi0Offset] = items[i].Index;
}
}
// finalize
// ========
m2m.Dispose();
return(inv);
}
}
