/// <summary>
/// MPI update of a bit-array
/// </summary>
/// <param name="b"></param>
/// <param name="GridData"></param>
static public void MPIExchange(this BitArray b, IGridData GridData)
{
if (b.Length != GridData.iLogicalCells.NoOfCells)
{
throw new ArgumentException("length must be equal to number of cells.", "b");
}
if (GridData.CellPartitioning.MpiSize > 1)
{
// external
{
int rank, size;
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank);
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out size);
// setup messenger
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
sms.SetCommPathsAndCommit(GridData.iParallel.ProcessesToSendTo);
// send data
for (int p = 0; p < size; p++)
{
int[] sendlist = GridData.iParallel.SendCommLists[p];
if (sendlist == null)
{
continue;
}
int L = sendlist.Length;
System.Collections.BitArray packet_for_p = new System.Collections.BitArray(L, false);
for (int l = 0; l < L; l++)
{
packet_for_p[l] = b[sendlist[l]];
}
sms.Transmitt(p, packet_for_p);
}
// receive data
System.Collections.BitArray rcv_dat;
int rcv_rank;
while (sms.GetNext(out rcv_rank, out rcv_dat))
{
int insertAt = GridData.iParallel.RcvCommListsInsertIndex[rcv_rank];
if (GridData.iParallel.RcvCommListsNoOfItems[rcv_rank] != rcv_dat.Count)
{
throw new ApplicationException("internal error.");
}
int C = rcv_dat.Count;
for (int i = 0; i < C; i++)
{
b[insertAt + i] = rcv_dat[i];
}
}
// dispose
sms.Dispose();
}
}
}
/// <summary>
/// ctor.
/// </summary>
/// <param name="M"></param>
/// <param name="ExtCol">
/// key: processor rank 'p' <br/>
/// value: a list of column indices (within the local range of columns of <paramref name="M"/>),
/// which should be editable at rank 'p'.
/// </param>
public MsrExtMatrix(IMutableMatrixEx M, IDictionary <int, int[]> ExtCol)
{
this.ColPart = M.ColPartition;
int i0Row = (int)M.RowPartitioning.i0, I = M.RowPartitioning.LocalLength,
i0Col = (int)this.ColPart.i0, J = this.ColPart.LocalLength;
Mtx = M;
// init
// ====
ColToRowLocal = new List <int> [ColPart.LocalLength];
for (int j = 0; j < ColToRowLocal.Length; j++)
{
ColToRowLocal[j] = new List <int>();
}
// build Column to row - mapping
// =============================
ColToRowExternal = new Dictionary <int, List <int> >();
// key: global column index j, within the range of processor 'p'
// values: global row indices
SortedDictionary <int, List <int> > ColForProc = new SortedDictionary <int, List <int> >();
// key: MPI processor index 'p'
// values: a set of global column indices, within the range of processor 'p',
// that contain nonzero entries on this processor
int[] col = null;
int L;
// loop over all rows...
for (int i = 0; i < I; i++)
{
L = M.GetOccupiedColumnIndices(i + i0Row, ref col);
// loop over all nonzero entries in the row...
for (int l = 0; l < L; l++)
{
int ColIndex = col[l];
int localColInd = ColIndex - i0Col;
if (localColInd >= 0 && localColInd < J)
{
// column of 'entry' is within the local range of this processor
// + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColToRowLocal[localColInd].Add(i + i0Row);
}
else
{
// column of 'entry' belongs to external processor 'proc'
// + + + + + + + + + + + + + + + + + + + + + + + + + + + +
int proc = this.ColPart.FindProcess(ColIndex);
{
//SortedDictionary<int, List<int>> ColToRowExt_proc;
//if (!ColToRowExternal.ContainsKey(proc)) {
// ColToRowExt_proc = new SortedDictionary<int, List<int>>();
// ColToRowExternal.Add(proc, ColToRowExt_proc);
//} else {
// ColToRowExt_proc = ColToRowExternal[proc];
//}
int j = ColIndex;
List <int> Rows4Col;
if (!ColToRowExternal.ContainsKey(j))
{
Rows4Col = new List <int>();
ColToRowExternal.Add(j, Rows4Col);
}
else
{
Rows4Col = ColToRowExternal[j];
}
Rows4Col.Add(i + i0Row);
}
{
List <int> ColForProc_proc;
if (!ColForProc.ContainsKey(proc))
{
ColForProc_proc = new List <int>();
ColForProc.Add(proc, ColForProc_proc);
}
else
{
ColForProc_proc = ColForProc[proc];
}
if (!ColForProc_proc.Contains(ColIndex))
{
ColForProc_proc.Add(ColIndex);
}
}
}
}
}
// communicate
// ===========
//SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw.MPI_COMM_WORLD);
//{
// foreach (int proc in ColToRowExternal.Keys) {
// sms.SetCommPath(proc);
// }
// sms.CommitCommPaths();
// // send
// foreach (int proc in ColToRowExternal.Keys) {
// SortedDictionary<int, List<int>> ColToRowExt_proc = ColToRowExternal[proc];
// SortedList _ColToRowExt_proc = new SortedList();
// foreach (int iCol in ColToRowExt_proc.Keys)
// _ColToRowExt_proc.Add(iCol, ColToRowExt_proc[iCol].ToArray());
// }
// // receive
// int p; SortedList rcv;
// sms.GetNext(out p, out rcv);
// while (rcv != null) {
// foreach (int col in rcv.Keys) {
// int[] rowList = (int[])rcv[col];
// ColToRowLocal[col].AddRange(rowList);
// }
// sms.GetNext(out p, out rcv);
// }
//}
//sms.Dispose();
// build 'ColProcessors'
// =====================
//ColProcessors = new List<int>[ColToRowLocal.Length];
//for (int j = 0; j < ColToRowLocal.Length; j++) {
// List<int> mpiRank = null;
// foreach (int rowind in ColToRowLocal[j]) {
// int riloc = rowind - i0Row;
// if (riloc < 0 || riloc >= I) {
// if (mpiRank == null)
// mpiRank = new List<int>();
// mpiRank.Add(M.RowPartiton.FindProcess(rowind));
// }
// ColProcessors[j] = mpiRank;
// }
//}
// communicate: build 'ColProcessors'
// ==================================
{
ColProcessors = new List <int> [ColPart.LocalLength];
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
sms.SetCommPathsAndCommit(ColForProc.Keys);
foreach (int proc in ColForProc.Keys)
{
sms.Transmit(proc, ColForProc[proc].ToArray());
}
int i0Loc = (int)ColPart.i0;
int rcvproc; int[] ColIndices;
while (sms.GetNext(out rcvproc, out ColIndices))
{
int Rank;
{
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out Rank);
//Console.WriteLine("P# " + Rank + ": receiving from P# " + rcvproc);
}
foreach (int ColInd in ColIndices)
{
int localColInd = ColInd - i0Loc;
if (localColInd < 0 || localColInd >= ColPart.LocalLength)
{
throw new IndexOutOfRangeException("internal error");
}
if (ColProcessors[localColInd] == null)
{
ColProcessors[localColInd] = new List <int>();
}
if (ColProcessors[localColInd].Contains(rcvproc))
{
throw new ApplicationException("internal error.");
}
ColProcessors[localColInd].Add(rcvproc);
}
}
sms.Dispose();
}
if (ExtCol != null)
{
var send = new Dictionary <int, List <Tuple <int, List <int> > > >();
int myRank = M.RowPartitioning.MpiRank;
foreach (var kv in ExtCol)
{
int rank = kv.Key;
int[] ColIdx = kv.Value;
var sendToRank = new List <Tuple <int, List <int> > >();
foreach (int iCol in ColIdx)
{
List <int> c2p = ColProcessors[iCol - i0Col];
var t = new Tuple <int, List <int> >(iCol, c2p != null ? new List <int>(c2p) : new List <int>());
t.Item2.Add(myRank);
sendToRank.Add(t);
}
send.Add(rank, sendToRank);
}
var receive = SerialisationMessenger.ExchangeData(send, csMPI.Raw._COMM.WORLD);
ColProcessorsExternal = new Dictionary <int, List <int> >();
foreach (var kv in receive)
{
var val = kv.Value;
foreach (var t in val)
{
int iCol = t.Item1;
List <int> ranks = t.Item2;
int iMyRank = ranks.IndexOf(myRank);
if (iMyRank >= 0)
{
ranks.RemoveAt(iMyRank);
}
ColProcessorsExternal.Add(t.Item1, t.Item2);
Debug.Assert(this.ColPart.FindProcess(t.Item1) == kv.Key);
}
}
#if DEBUG
foreach (var procList in ColProcessorsExternal.Values)
{
Debug.Assert(procList.Contains(myRank) == false);
}
#endif
}
}
