/// <summary>
/// ctor.
/// </summary>
/// <param name="localsize">
/// number of entries that should be stored in this MPI process
/// </param>
/// <param name="c">
/// MPI communicator, <see cref="MPI_Comm"/>.
/// </param>
/// <param name="_BlockSize">
/// See <see cref="BlockSize"/>.
/// </param>
public Partitioning(int localsize, MPI.Wrappers.MPI_Comm c)
{
m_comm = c;
MPICollectiveWatchDog.Watch(c);
csMPI.Raw.Comm_Rank(c, out m_rank);
csMPI.Raw.Comm_Size(c, out m_size);
m_LocalLengths = new int[m_size];
int[] ll = { localsize };
unsafe
{
fixed(void *pSndBuf = ll, pRcvBuf = m_LocalLengths)
{
csMPI.Raw.Allgather((IntPtr)pSndBuf, 4, csMPI.Raw._DATATYPE.BYTE,
(IntPtr)pRcvBuf, 4, csMPI.Raw._DATATYPE.BYTE,
m_comm);
}
}
m_i0Offset = new int[m_size + 1];
m_i0Offset[0] = 0;
for (int i = 1; i <= m_size; i++)
{
long _i0oL = (long)(m_i0Offset[i - 1]) + (long)(m_LocalLengths[i - 1]);
if (_i0oL > int.MaxValue)
{
throw new OverflowException("Partition exceeds the range of 32-bit integer.");
}
m_i0Offset[i] = m_i0Offset[i - 1] + m_LocalLengths[i - 1];
}
}
/// <summary>
/// ctor.
/// </summary>
/// <param name="localsize">
/// number of entries that should be stored in this MPI process
/// </param>
/// <param name="c">
/// MPI communicator, <see cref="MPI_Comm"/>.
/// </param>
/// <param name="_BlockSize">
/// See <see cref="BlockSize"/>.
/// </param>
public Partitioning(int localsize, MPI.Wrappers.MPI_Comm c)
{
m_comm = c;
MPICollectiveWatchDog.Watch(c);
csMPI.Raw.Comm_Rank(c, out m_rank);
csMPI.Raw.Comm_Size(c, out m_size);
m_LocalLengths = new int[m_size];
int[] ll = { localsize };
unsafe
{
fixed(void *pSndBuf = ll, pRcvBuf = m_LocalLengths)
{
csMPI.Raw.Allgather((IntPtr)pSndBuf, 4, csMPI.Raw._DATATYPE.BYTE,
(IntPtr)pRcvBuf, 4, csMPI.Raw._DATATYPE.BYTE,
m_comm);
}
}
m_i0Offset = new int[m_size + 1];
m_i0Offset[0] = 0;
for (int i = 1; i <= m_size; i++)
{
m_i0Offset[i] = m_i0Offset[i - 1] + m_LocalLengths[i - 1];
}
}
/// <summary>
/// returns the minimum of each entry of <paramref name="i"/> on all MPI-processes in the
/// <paramref name="comm"/>--communicator.
/// </summary>
static public int[] MPIMin(this int[] i, MPI_Comm comm)
{
int[] R = new int[i.Length];
unsafe
{
fixed(int *loc = i, glob = R)
{
csMPI.Raw.Allreduce(((IntPtr)(loc)), ((IntPtr)(glob)), i.Length, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MIN, comm);
}
}
return(R);
}
/// <summary>
/// returns the minimum of each entry of <paramref name="i"/> on all MPI-processes in the
/// <paramref name="comm"/>--communicator.
/// </summary>
static public double[] MPIMin(this double[] i, MPI_Comm comm)
{
double[] R = new double[i.Length];
unsafe
{
fixed(double *loc = i, glob = R)
{
csMPI.Raw.Allreduce(((IntPtr)(loc)), ((IntPtr)(glob)), i.Length, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.MIN, comm);
}
}
return(R);
}
/// <summary>
/// Returns, for each entry, the sum <paramref name="A"/> on all MPI-processes in the
/// <paramref name="comm"/>--communicator.
/// </summary>
static public int[] MPISum(this int[] A, MPI_Comm comm)
{
int[] S = new int[A.Length];
unsafe
{
fixed(int *pA = A, pS = S)
{
csMPI.Raw.Allreduce(((IntPtr)(pA)), ((IntPtr)(pS)), A.Length, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.SUM, comm);
}
return(S);
}
}
/// <summary>
/// Returns, for each entry, the sum <paramref name="A"/> on all MPI-processes in the
/// <paramref name="comm"/>--communicator.
/// </summary>
static public double[] MPISum(this double[] A, MPI_Comm comm)
{
double[] S = new double[A.Length];
unsafe
{
fixed(double *pA = A, pS = S)
{
csMPI.Raw.Allreduce(((IntPtr)(pA)), ((IntPtr)(pS)), 1, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.SUM, comm);
}
return(S);
}
}
/// <summary>
///
/// </summary>
public byte[] Comm_f2c(MPI_Comm F_Comm)
{
byte[] ret = new byte[this.GetSizeof_C_MPI_comm()];
unsafe {
byte *pComm = (byte *)&(F_Comm.m1);
for (int i = 0; i < ret.Length; i++)
{
ret[i] = *pComm;
pComm++;
}
}
return(ret);
}
/// <summary>
/// Wrapper around <see cref="IMPIdriver.Scatterv(IntPtr, IntPtr, IntPtr, MPI_Datatype, IntPtr, int, MPI_Datatype, int, MPI_Comm)"/>.
/// </summary>
static public int[] MPIScatterv(this int[] send, int[] sendcounts, int root, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(comm, out int size);
csMPI.Raw.Comm_Rank(comm, out int rank);
int[] result = new int[Math.Max(1, sendcounts[rank])];
unsafe {
int *displs = stackalloc int[size];
for (int i = 1; i < size; i++)
{
displs[i] = displs[i - 1] + sendcounts[i - 1];
}
if (rank == root)
{
if (send.Length < displs[size - 1] + sendcounts[size - 1])
{
throw new ArgumentException("Mismatch between send counts and send buffer size.");
}
}
//if (send == null || send.Length == 0) {
// // Dummy to avoid null pointer exception
// send = new int[1];
//}
fixed(int *pSend = send, pSendcounts = &sendcounts[0], pResult = &result[0])
{
csMPI.Raw.Scatterv(
(IntPtr)pSend,
(IntPtr)pSendcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.INT,
(IntPtr)pResult,
sendcounts[rank],
csMPI.Raw._DATATYPE.INT,
root,
comm);
}
}
if (result.Length != sendcounts[rank])
{
Debug.Assert(result.Length == 1);
Debug.Assert(sendcounts[rank] == 0);
Array.Resize(ref result, 0);
}
return(result);
}
/// <summary>
/// returns the maximum of <paramref name="i"/> on all MPI-processes in the
/// <paramref name="comm"/>--communicator.
/// </summary>
static public int MPIMax(this int i, MPI_Comm comm)
{
int loc = i;
unsafe {
int glob = int.MinValue;
csMPI.Raw.Allreduce(
(IntPtr)(&loc),
(IntPtr)(&glob),
1,
csMPI.Raw._DATATYPE.INT,
csMPI.Raw._OP.MAX,
comm);
return(glob);
}
}
/// <summary>
/// returns true on every process for every entry of <paramref name="i"/> if they are equal at every process
/// </summary>
/// <param name="i"></param>
/// <param name="comm"></param>
/// <returns></returns>
static public bool[] MPIEquals(this int[] i, MPI_Comm comm)
{
int[] R = new int[i.Length];
bool[] check = new bool[i.Length];
unsafe
{
fixed(int *loc = i, glob = R)
{
csMPI.Raw.Allreduce(((IntPtr)(loc)), ((IntPtr)(glob)), i.Length, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.BXOR, comm);
}
}
for (int k = 0; k < i.Length; k++)
{
check[k] = R[k] == 0 ? true : false;
}
return(check);
}
/// <summary>
/// MPI-process with rank <paramref name="root"/> gathers this int[] of all MPI-processes in the
/// <paramref name="comm"/>-communicator with variable length. The length of the gathered int[] is specified by <paramref name="recvcount"/>
/// </summary>
static public int[] MPIGatherv(this int[] send, int[] recvcounts, int root, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(comm, out int size);
csMPI.Raw.Comm_Rank(comm, out int rank);
int rcs = rank == root?recvcounts.Sum() : 0;
int[] result = rank == root ? new int[Math.Max(1, rcs)] : null;
unsafe {
int *displs = stackalloc int[size];
if (rank == root)
{
for (int i = 1; i < size; i++)
{
displs[i] = displs[i - 1] + recvcounts[i - 1];
}
}
fixed(int *pSend = send, pRcvcounts = recvcounts, pResult = result)
{
Debug.Assert((rank == root) != (pResult == null));
csMPI.Raw.Gatherv(
(IntPtr)pSend,
send.Length,
csMPI.Raw._DATATYPE.INT,
(IntPtr)pResult,
(IntPtr)pRcvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.INT,
root,
comm);
}
}
if (result != null && result.Length > rcs)
{
Debug.Assert(rcs == 0);
result = new int[0];
}
return(result);
}
/// <summary>
/// Gathers all send Arrays on all MPI-processes, at which every jth block of data is from the jth process.
/// </summary>
static private int[] Int_MPIAllGatherv(this int[] send, int[] m_recvcounts, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out int size);
int rcs = m_recvcounts.Sum();
if (rcs == 0)
{
return(new int[0]);
}
int[] result = new int[rcs];
if (send.Length == 0)
{
send = new int[1];
}
unsafe {
int *displs = stackalloc int[size];
for (int i = 1; i < size; i++)
{
displs[i] = displs[i - 1] + m_recvcounts[i - 1];
}
fixed(int *pResult = result, pSend = send)
{
fixed(int *pRcvcounts = m_recvcounts)
{
csMPI.Raw.Allgatherv(
(IntPtr)pSend,
send.Length,
csMPI.Raw._DATATYPE.INT,
(IntPtr)pResult,
(IntPtr)pRcvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.INT,
comm);
}
}
}
return(result);
}
private void LoadSymb(string name, ref MPI_Comm sym)
{
string errstr;
IntPtr addr = Utils.DynamicLibraries.LoadSymbol(m_conv.LibHandle, name, out errstr);
if (addr == IntPtr.Zero)
{
throw new ApplicationException("OpenMPI error: unable to load symbol '" + name + "' from library '" + m_conv.CurrentLibraryName + "', Error string: >" + errstr + "<;");
}
sym.m1 = m_conv.MPI_Comm_c2f(addr);
//Console.WriteLine("val of '" + name + "' is: " + addr + ", fortran value is " + sym.m1);
IntPtr test = m_conv.MPI_Comm_f2c(sym);
if (test != addr)
{
throw new ApplicationException("f**k");
}
//else
// Console.WriteLine("back passed.");
}
/// <summary>
/// ctor
/// </summary>
/// <param name="comm"></param>
//// <typeparam name="t">the primitive type of the message; must be a value-type;</typeparam>
public Many2ManyMessenger(MPI.Wrappers.MPI_Comm comm)
{
m_Comm = comm;
MPI.Wrappers.csMPI.Raw.Comm_Size(m_Comm, out size);
MPI.Wrappers.csMPI.Raw.Comm_Rank(m_Comm, out MyRank);
m_MyCommPaths = new int[size];
m_AllCommPaths = new int[size, size];
m_ReceiveBuffers = new Buffer[size];
m_SendBuffers = new Buffer[size];
m_Requests = new MPI_Request[size * 2];
m_ArrayOfStatuses = new MPI_Status[m_Requests.Length];
// check item type
m_ItemType = typeof(ItemType);
CheckItemTypeRecursive(m_ItemType);
m_ItemSize = Marshal.SizeOf(m_ItemType);
}
static public int[] MPIAllGather(this int i, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out int size);
int[] result = new int[size];
unsafe {
int sendBuffer = i;
fixed(int *pResult = &result[0])
{
csMPI.Raw.Allgather(
(IntPtr)(&i),
1,
csMPI.Raw._DATATYPE.INT,
(IntPtr)pResult,
1,
csMPI.Raw._DATATYPE.INT,
comm);
}
}
return(result);
}
static public double[] MPIAllGather(this double d, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out int size);
double[] result = new double[size];
unsafe {
double sendBuffer = d;
fixed(double *pResult = &result[0])
{
csMPI.Raw.Allgather(
(IntPtr)(&d),
1,
csMPI.Raw._DATATYPE.DOUBLE,
(IntPtr)pResult,
1,
csMPI.Raw._DATATYPE.DOUBLE,
comm);
}
}
return(result);
}
/// <summary>
/// returns true on every process for every entry of <paramref name="i"/> if they are equal at every process
/// </summary>
/// <param name="i"></param>
/// <param name="comm"></param>
/// <returns></returns>
static public bool[] MPIEquals(this double[] i, MPI_Comm comm)
{
ulong[] conv_loc = new ulong[i.Length];
for (int iDbl = 0; iDbl < i.Length; iDbl++)
{
conv_loc[iDbl] = (ulong)i[iDbl];
}
ulong[] R = new ulong[i.Length];
bool[] check = new bool[i.Length];
unsafe
{
fixed(ulong *loc = conv_loc, glob = R)
{
csMPI.Raw.Allreduce(((IntPtr)(loc)), ((IntPtr)(glob)), i.Length, csMPI.Raw._DATATYPE.UNSIGNED_LONG_LONG, csMPI.Raw._OP.BXOR, comm);
}
}
for (int k = 0; k < i.Length; k++)
{
check[k] = R[k] == 0? true : false;
}
return(check);
}
/// <summary>
///
/// </summary>
public MPI_Comm Comm_c2f(byte[] C_MPI_Comm)
{
if (C_MPI_Comm.Length != GetSizeof_C_MPI_comm())
{
throw new ArgumentException("wrong number of bytes; length of argument must match the value that is returend by 'GetSizeof_C_MPI_comm()';");
}
MPI_Comm F_Comm = default(MPI_Comm);
unsafe
{
fixed(byte *pcComm = &C_MPI_Comm[0])
{
byte *pComm = (byte *)&(F_Comm.m1);
for (int i = 0; i < C_MPI_Comm.Length; i++)
{
*pComm = pcComm[i];
pComm++;
}
}
}
return(F_Comm);
}
/// <summary>
/// converts the MPI communicators in ilPSP (which are
/// FORTRAN-communicators) into a C-Communicator
/// </summary>
/// <param name="EightByteComm">
/// on exit, the value of the C - MPI communicator handle, if
/// <see cref="IMPI_CommConstants.GetSizeof_C_MPI_comm"/> == 8,
/// otherwise 0
/// </param>
/// <param name="FourByteComm">
/// on exit, the value of the C - MPI communicator handle, if
/// <see cref="IMPI_CommConstants.GetSizeof_C_MPI_comm"/> == 4,
/// otherwise 0;
/// </param>
/// <param name="input">
/// (FORTRAN) MPI communicator handle to convert
/// </param>
/// <returns>
/// Equal to <see cref="IMPI_CommConstants.GetSizeof_C_MPI_comm"/>,
/// either 4 or 8
/// </returns>
/// <remarks>
/// The length of the C - MPI communicator handle depends on the used
/// MPI implementation (see remarks at <see cref="MPI_Comm"/>) and is
/// given by to the return value of
/// <see cref="IMPI_CommConstants.GetSizeof_C_MPI_comm"/>
/// </remarks>
public int MPI_Comm_f2c(MPI_Comm input, out uint FourByteComm, out ulong EightByteComm)
{
unsafe {
byte[] C_Comm = _COMM.Comm_f2c(input);
fixed(byte *pComm = &(C_Comm[0]))
{
uint _FourByteComm = 0;
ulong _EightByteComm = 0;
byte *pDest;
switch (C_Comm.Length)
{
case 4:
pDest = (byte *)(&_FourByteComm);
break;
case 8:
pDest = (byte *)(&_EightByteComm);
break;
default:
throw new NotImplementedException(
"unknown size of C MPI communicator: " + C_Comm.Length + "bytes; ");
}
for (int i = 0; i < C_Comm.Length; i++)
{
pDest[i] = pComm[i];
}
FourByteComm = _FourByteComm;
EightByteComm = _EightByteComm;
}
return(C_Comm.Length);
}
}
static public int[] MPIScatterv(this int[] send, int[] sendcounts, int root, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out int size);
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out int rank);
int[] result = new int[sendcounts[rank]];
unsafe {
int *displs = stackalloc int[size];
for (int i = 1; i < size; i++)
{
displs[i] = displs[i - 1] + sendcounts[i - 1];
}
if (send == null || send.Length == 0)
{
// Dummy to avoid null pointer exception
send = new int[1];
}
fixed(int *pSend = &send[0], pSendcounts = &sendcounts[0], pResult = &result[0])
{
csMPI.Raw.Scatterv(
(IntPtr)pSend,
(IntPtr)pSendcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.INT,
(IntPtr)pResult,
sendcounts[rank],
csMPI.Raw._DATATYPE.INT,
root,
comm);
}
}
return(result);
}
/// <summary>
/// MPI - parallel scalar product
/// </summary>
static public double ddot <TX, TY>(int N, TX x, int incx, TY y, int incy, MPI.Wrappers.MPI_Comm comm)
where TX : IList <double>
where TY : IList <double>
{
if (incx * x.Count < N)
{
throw new ArgumentException("vector too short.", "x");
}
if (incy * y.Count < N)
{
throw new ArgumentException("vector too short.", "y");
}
double locRes = 0;
double[] dx = x as double[];
double[] dy = y as double[];
if (dx != null && dy != null)
{
// use dnese BLAS
locRes = BLAS.ddot(N, dx, incx, dy, incy);
}
else
{
ISparseVector <double> spx = x as ISparseVector <double>;
ISparseVector <double> spy = y as ISparseVector <double>;
IList <double> _y = y;
if (spy != null)
{
if (spx == null || spy.Sparsity < spx.Sparsity)
{
// y is sparser than x, use y !
spx = spy;
spy = null;
_y = x;
x = default(TX);
y = default(TY);
int buffy = incx;
incx = incy;
incy = buffy;
}
}
if (spx != null)
{
// sparse implementation
foreach (var entry in spx.SparseStruct)
{
int m = entry.Key % incx;
if (m != 0)
{
// entry is skipped by x-increment
continue;
}
int n = entry.Key / incx;
locRes += entry.Value * _y[n * incy];
}
}
else
{
// default IList<double> - implementation
for (int n = 0; n < N; n++)
{
locRes += x[n * incx] * y[n * incy];
}
}
}
double globRes = double.NaN;
unsafe {
csMPI.Raw.Allreduce((IntPtr)(&locRes), (IntPtr)(&globRes), 1, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.SUM, comm);
}
return(globRes);
}
/// <summary>
/// Wrapper around <see cref="IMPIdriver.Scatterv(IntPtr, IntPtr, IntPtr, MPI_Datatype, IntPtr, int, MPI_Datatype, int, MPI_Comm)"/>.
/// </summary>
static public double[] MPIScatterv(this double[] send, int[] sendcounts, int root, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(comm, out int size);
csMPI.Raw.Comm_Rank(comm, out int rank);
double[] result = new double[sendcounts[rank]];
unsafe
{
int *displs = stackalloc int[size];
for (int i = 1; i < size; i++)
{
displs[i] = displs[i - 1] + sendcounts[i - 1];
//sum += sendcounts[i];
}
if (rank == root)
{
if (send.Length < displs[size - 1] + sendcounts[size - 1])
{
throw new ArgumentException("Mismatch between send counts and send buffer size.");
}
}
//if (send == null || send.Length == 0) {
// // Dummy to avoid null pointer exception
// send = new double[1];
//}
fixed(int *pSendcounts = &sendcounts[0])
{
fixed(double *pSend = send, pResult = &result[0])
{
csMPI.Raw.Scatterv(
(IntPtr)pSend,
(IntPtr)pSendcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.DOUBLE,
(IntPtr)pResult,
sendcounts[rank],
csMPI.Raw._DATATYPE.DOUBLE,
root,
comm);
}
}
}
return(result);
}
/// <summary>
/// Gathers objects <paramref name="o"/> from each rank on rank <paramref name="root"/>.
/// </summary>
/// <param name="o"></param>
/// <param name="root">
/// MPI rank where the objects should be collected
/// </param>
/// <param name="comm"></param>
/// <returns>
/// - null on all ranks except <paramref name="root"/>
/// - on rank <paramref name="root"/>, an array containing the objects from all ranks
/// </returns>
static public T[] MPIGatherO <T>(this T o, int root, MPI_Comm comm)
{
csMPI.Raw.Comm_Rank(comm, out int MyRank);
csMPI.Raw.Comm_Size(comm, out int MpiSize);
IFormatter _Formatter = new BinaryFormatter();
// -----------------------------------------------------
// 1st phase: serialize object and gather object size
// -----------------------------------------------------
byte[] buffer = null;
int Size;
if (root != MyRank)
{
using (var ms = new MemoryStream()) {
_Formatter.Serialize(ms, o);
Size = (int)ms.Position;
buffer = ms.GetBuffer();
}
if (Size <= 0)
{
throw new IOException("Error serializing object for MPI broadcast - size is 0");
}
Array.Resize(ref buffer, Size);
}
else
{
buffer = new byte[0];
Size = 0;
}
int[] Sizes = Size.MPIGather(root, comm);
// -----------------------------------------------------
// 2nd phase: gather data
// -----------------------------------------------------
byte[] rcvBuffer = buffer.MPIGatherv(Sizes);
// -----------------------------------------------------
// 3rd phase: de-serialize
// -----------------------------------------------------
if (MyRank == root)
{
T[] ret = new T[MpiSize];
using (var ms = new MemoryStream(rcvBuffer)) {
for (int r = 0; r < MpiSize; r++)
{
if (r == MyRank)
{
ret[r] = o;
}
else
{
ret[r] = (T)_Formatter.Deserialize(ms);
}
}
}
return(ret);
}
else
{
return(null);
}
}
/// <summary>
/// Wrapper around <see cref="IMPIdriver.Gatherv(IntPtr, int, MPI_Datatype, IntPtr, IntPtr, IntPtr, MPI_Datatype, int, MPI_Comm)"/>
/// </summary>
static public double[] MPIGatherv(this double[] send, int[] recvcounts, int root, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(comm, out int size);
csMPI.Raw.Comm_Rank(comm, out int rank);
double[] result = rank == root ? new double[recvcounts.Sum()] : null;
unsafe {
int *displs = stackalloc int[size];
for (int i = 1; i < size; i++)
{
displs[i] = displs[i - 1] + recvcounts[i - 1];
}
fixed(int *pRcvcounts = &recvcounts[0])
{
fixed(double *pSend = &send[0], pResult = result)
{
Debug.Assert((rank == root) != (pResult == null));
csMPI.Raw.Gatherv(
(IntPtr)pSend,
send.Length,
csMPI.Raw._DATATYPE.DOUBLE,
(IntPtr)pResult,
(IntPtr)pRcvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.DOUBLE,
root,
comm);
}
}
}
return(result);
}
/// <summary>
/// MPI-process with rank <paramref name="root"/> gathers this ulong[] of all MPI-processes in the
/// <paramref name="comm"/>-communicator with variable length. The length of the gathered long[] is specified by <paramref name="recvcount"/>
/// </summary>
/// <param name="recvcount">
/// Length of the receive buffer
/// </param>
static public ulong[] MPIGatherv(this ulong[] send, int[] recvcount, int root, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(comm, out int size);
ulong[] result = new ulong[recvcount.Sum()];
unsafe {
int *displs = stackalloc int[size];
for (int i = 1; i < size; i++)
{
displs[i] = displs[i - 1] + recvcount[i - 1];
}
//LONG_LONG for long of 64 bits in size
fixed(ulong *pSend = &send[0], pResult = &result[0])
{
fixed(int *pRcvcounts = &recvcount[0])
{
csMPI.Raw.Gatherv(
(IntPtr)pSend,
send.Length,
csMPI.Raw._DATATYPE.LONG_LONG,
(IntPtr)pResult,
(IntPtr)pRcvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.LONG_LONG,
root,
comm);
}
}
}
return(result);
}
/// <summary>
/// Gathers all send Arrays on all MPI-processes, at which every jth block of data is from the jth process.
/// </summary>
static private ulong[] Long_MPIAllGatherv(this ulong[] send, int[] m_recvcounts, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out int size);
ulong[] result = new ulong[m_recvcounts.Sum()];
unsafe {
int *displs = stackalloc int[size];
for (int i = 1; i < size; i++)
{
displs[i] = displs[i - 1] + m_recvcounts[i - 1];
}
fixed(ulong *pResult = &result[0], pSend = &send[0])
{
fixed(int *pRcvcounts = &m_recvcounts[0])
{
csMPI.Raw.Allgatherv(
(IntPtr)pSend,
send.Length,
csMPI.Raw._DATATYPE.UNSIGNED_LONG_LONG,
(IntPtr)pResult,
(IntPtr)pRcvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.UNSIGNED_LONG_LONG,
comm);
}
}
}
return(result);
}
/// <summary>
/// broadcasts an object to all other processes in the
/// MPI communicator <paramref name="comm"/>.
/// </summary>
/// <param name="o">
/// an arbitrary, serialize able object;
/// ignored on all processes which are not <paramref name="root"/>
/// </param>
/// <param name="root">
/// rank of the sender process
/// </param>
/// <param name="comm">MPI communicator</param>
/// <returns>
/// on the sender process, the input <paramref name="o"/> is returned;
/// </returns>
/// <typeparam name="T">
/// type of the object to transmit
/// </typeparam>
static public T MPIBroadcast <T>(this T o, int root, MPI_Comm comm)
{
int MyRank;
csMPI.Raw.Comm_Rank(comm, out MyRank);
IFormatter _Formatter = new BinaryFormatter();
// -----------------------------------------------------
// 1st phase: serialize object and broadcast object size
// -----------------------------------------------------
byte[] buffer = null;
int Size = -1;
if (root == MyRank)
{
using (var ms = new MemoryStream()) {
_Formatter.Serialize(ms, o);
Size = (int)ms.Position;
buffer = ms.GetBuffer();
}
if (Size <= 0)
{
throw new IOException("Error serializing object for MPI broadcast - size is 0");
}
Array.Resize(ref buffer, Size);
}
unsafe {
csMPI.Raw.Bcast((IntPtr)(&Size), 4, csMPI.Raw._DATATYPE.BYTE, root, comm);
}
// ---------------------------
// 2nd phase: broadcast object
// ---------------------------
Debug.Assert(Size > 0);
if (buffer == null)
{
buffer = new byte[Size];
}
unsafe
{
fixed(byte *pBuffer = buffer)
{
csMPI.Raw.Bcast((IntPtr)pBuffer, Size, csMPI.Raw._DATATYPE.BYTE, root, comm);
}
}
if (MyRank == root)
{
return(o);
}
else
{
T r;
using (var ms = new MemoryStream(buffer)) {
r = (T)_Formatter.Deserialize(ms);
return(r);
}
}
}
/// <summary>
/// MPI-process with rank <paramref name="root"/> gathers this ulong[] of all MPI-processes in the
/// <paramref name="comm"/>-communicator with variable length. The length of the gathered long[] is specified by <paramref name="recvcount"/>
/// </summary>
/// <param name="recvcount">
/// number of items to receive from each sender
/// </param>
/// <param name="send">
/// data to send
/// </param>
/// <param name="comm"></param>
/// <param name="root">rank of receiver process</param>
static public ulong[] MPIGatherv(this ulong[] send, int[] recvcount, int root, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(comm, out int size);
csMPI.Raw.Comm_Rank(comm, out int rank);
int rcs = rank == root?recvcount.Sum() : 0;
ulong[] result = rank == root ? new ulong[Math.Max(1, rcs)] : null;
unsafe {
int *displs = stackalloc int[size];
if (rank == root)
for (int i = 1; i < size; i++)
{
displs[i] = displs[i - 1] + recvcount[i - 1];
}
//LONG_LONG for long of 64 bits in size
fixed(ulong *pSend = send, pResult = result)
{
fixed(int *pRcvcounts = recvcount)
{
csMPI.Raw.Gatherv(
(IntPtr)pSend,
send.Length,
csMPI.Raw._DATATYPE.LONG_LONG,
(IntPtr)pResult,
(IntPtr)pRcvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.LONG_LONG,
root,
comm);
}
}
}
if (result != null && result.Length > rcs)
{
Debug.Assert(rcs == 0);
result = new ulong[0];
}
return(result);
}
/// <summary>
/// Wrapper around <see cref="IMPIdriver.Gatherv"/>
/// </summary>
static public byte[] MPIGatherv(this byte[] send, int[] recvcounts, int root, MPI_Comm comm)
{
csMPI.Raw.Comm_Size(comm, out int size);
csMPI.Raw.Comm_Rank(comm, out int rank);
int outsize;
byte[] result;
if (rank == root)
{
outsize = recvcounts.Sum();
result = new byte[Math.Max(outsize, 1)]; // this is only because a 0-length array maps to unsafe null
}
else
{
result = null;
outsize = 0;
}
unsafe {
int *displs = stackalloc int[size];
if (rank == root)
for (int i = 1; i < size; i++)
{
displs[i] = displs[i - 1] + recvcounts[i - 1];
}
fixed(int *pRcvcounts = recvcounts)
{
int lsend = send.Length;
if (lsend <= 0)
{
send = new byte[0];
fixed(byte *pSend = send, pResult = result)
{
Debug.Assert((rank == root) != (pResult == null));
csMPI.Raw.Gatherv(
(IntPtr)pSend,
lsend,
csMPI.Raw._DATATYPE.BYTE,
(IntPtr)pResult,
(IntPtr)pRcvcounts,
(IntPtr)displs,
csMPI.Raw._DATATYPE.BYTE,
root,
comm);
}
}
}
if (outsize > 0)
{
Debug.Assert(outsize == result.Length);
return(result);
}
else
{
return(new byte[0]);
}
}
/// <summary>
/// if <paramref name="e"/> is unequal to null on any of the calling
/// MPI processes, this method broadcasts it to all other processes;
/// </summary>
/// <param name="e">
/// an exception, or null
/// </param>
/// <param name="comm">
/// %
/// </param>
/// <remarks>
/// The following code may cause a deadlock:
/// <code>
/// try {
/// // some code that may cause an exception on some MPI process,
/// // but not on all of them
/// } catch(Exception e) {
/// // some statement that have an effect on control flow, e.g. 'continue;' or
/// return false;
/// }
/// // some collective call, e.g.
/// csMPI.Raw.Allreduce( ... )
/// </code>
/// The collective call <c>csMPI.Raw.Allreduce( ... )</c> is never executed on some
/// MPI processes, therefore the application will ang in a deadlock.
/// <code>
/// try {
/// Exception e = null;
/// try {
/// // some code that may cause an exception on some MPI process,
/// // but not on all of them
/// } catch(Exception ee) {
/// e = ee;
/// }
/// ExceptionBcast(e,csMPI.Raw._COMM.WORLD);
/// } catch(Exception e2) {
/// // some statement that have an effect on control flow, e.g. 'continue;' or
/// return false;
/// }
/// // some collective call, e.g.
/// csMPI.Raw.Allreduce( ... )
/// </code>
/// Note that, if <paramref name="e"/> is not serializable,
/// only the message of <paramref name="e"/> will be broad-casted and
/// wrapped into a new exception.
/// </remarks>
static public void ExceptionBcast(this Exception e, MPI_Comm comm)
{
int ExcSrc = int.MaxValue;
if (e != null)
{
csMPI.Raw.Comm_Rank(comm, out ExcSrc);
}
unsafe {
int res = 0;
csMPI.Raw.Allreduce((IntPtr)(&ExcSrc), (IntPtr)(&res), 1, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MIN, comm);
ExcSrc = res;
}
if (ExcSrc < int.MaxValue)
{
// an exception occured on some process
int myRank;
csMPI.Raw.Comm_Rank(comm, out myRank);
object reduced;
if (myRank == ExcSrc)
{
// sender branch
if (e.GetType().GetCustomAttributes(typeof(SerializableAttribute), false).Length > 0)
{
// exception is serializeable -> bcast exception itself
reduced = MPIBroadcast <object>(e, ExcSrc, comm);
}
else
{
// bcast exception message
reduced = MPIBroadcast <object>(e.GetType().Name + ": '" + e.Message + "'", ExcSrc, comm);
}
}
else
{
// receiver branch
reduced = MPIBroadcast <object>(null, ExcSrc, comm);
}
if (reduced is string)
{
throw new ApplicationException("On MPI Process #" + myRank + ": " + ((string)reduced));
}
else if (reduced is Exception)
{
throw ((Exception)reduced);
}
else
{
throw new ApplicationException("should never occur");
}
}
}
