/// <summary>
/// MPI-parallel 2-norm
/// </summary>
static public double drnm2 <TX>(int N, TX x, int incx, MPI_Comm comm) where TX : IList <double>
{
double locRes = 0;
double[] dx = x as double[];
if (dx != null)
{
// double[] - implementation
locRes = BLAS.dnrm2(N, dx, incx);
locRes = locRes * locRes;
}
else
{
ISparseVector <double> spx = x as ISparseVector <double>;
if (spx != null)
{
// sparse implementation
foreach (var entry in spx.SparseStruct)
{
int m = entry.Key % incx;
if (m != 0)
{
// entry is skipped by x-increment
continue;
}
double xi = entry.Value;
locRes += xi * xi;
}
}
else
{
// default implementation
for (int n = 0; n < N; n++)
{
double xi = x[n * incx];
locRes += xi * xi;
}
}
}
double globRes = double.NaN;
unsafe {
csMPI.Raw.Allreduce((IntPtr)(&locRes), (IntPtr)(&globRes), 1, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.SUM, comm);
}
return(Math.Sqrt(globRes));
}
/// <summary>
/// Makes a "deep" copy of an attribute that is referenced by a pointer.
/// </summary>
private int DeepCopy(MPI_Comm comm, int keyval, IntPtr extra_state, IntPtr attribute_val_in,
IntPtr attribute_val_out, out int flag)
{
unsafe
{
IntPtr *outPtr = (IntPtr *)attribute_val_out;
*outPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(T)));
// Copy the bits
Marshal.StructureToPtr(Marshal.PtrToStructure(attribute_val_in, typeof(T)), *outPtr, false);
}
flag = 1;
return(Unsafe.MPI_SUCCESS);
}
/// <summary>
/// For parallel debugging
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="list"></param>
/// <param name="filename"></param>
public static void SaveToTextFileDebug <T>(this IEnumerable <T> list, string filename)
{
int Rank;
MPI_Comm comm = csMPI.Raw._COMM.WORLD;
csMPI.Raw.Comm_Rank(comm, out Rank);
string fullfilename = String.Concat(filename, "_", Rank);
using (var sw = new StreamWriter(fullfilename)) {
foreach (T element in list)
{
sw.WriteLine(element);
}
}
}
internal ValueArrayReceiveRequest(MPI_Comm comm, int source, int tag, T[] array)
{
this.cachedStatus = null;
this.array = array;
handle = GCHandle.Alloc(array, GCHandleType.Pinned);
// Initiate the non-blocking receive into "value"
int errorCode = Unsafe.MPI_Irecv(handle.AddrOfPinnedObject(), array.Length,
FastDatatypeCache <T> .datatype,
source, tag, comm, out request);
if (errorCode != Unsafe.MPI_SUCCESS)
{
handle.Free();
throw Environment.TranslateErrorIntoException(errorCode);
}
}
/// <summary>
/// Create a vector object
/// </summary>
public static int Create(MPI_Comm MPI_Comm, int jlower, int jupper, out T_IJVector vector)
{
ulong _com8;
uint _com4;
// we need to convert the MPI comm in ilPSP (which is a FORTRAN MPI comm)
// to a C-MPI comm: can be either 4 or 8 bytes!
int sz = csMPI.Raw.MPI_Comm_f2c(MPI_Comm, out _com4, out _com8);
switch (sz)
{
case 4: return(Create4(_com4, jlower, jupper, out vector));
case 8: return(Create8(_com8, jlower, jupper, out vector));
default: throw new NotImplementedException();
}
}
internal ValueReceiveRequest(MPI_Comm comm, int source, int tag, Action <T> action = null)
{
this.action = action;
this.cachedStatus = null;
this.value = default(T);
handle = GCHandle.Alloc(value, GCHandleType.Pinned);
unsafe
{
// Initiate the non-blocking receive into "value"
int errorCode = Unsafe.MPI_Irecv(handle.AddrOfPinnedObject(), 1, FastDatatypeCache <T> .datatype, source, tag, comm, out request);
if (errorCode != Unsafe.MPI_SUCCESS)
{
handle.Free();
throw Environment.TranslateErrorIntoException(errorCode);
}
}
}
/// <summary>
/// Create a solver object
/// </summary>
public static int HYPRE_ParaSailsCreate(MPI_Comm MPI_Comm, out T_Solver solver)
{
ulong _com8;
uint _com4;
// we need to convert the MPI comm in ilPSP (which is a FORTRAN MPI comm)
// to a C-MPI comm: can be either 4 or 8 bytes!
int sz = csMPI.Raw.MPI_Comm_f2c(MPI_Comm, out _com4, out _com8);
switch (sz)
{
case 4: return(Create4(_com4, out solver));
case 8: return(Create8(_com8, out solver));
default: throw new NotImplementedException();
}
}
/// <summary>
/// Generates Block Mask (index lists) from Sub block selection based on a multigrid mapping.
/// abstract parts are individuallized by child classes: <see cref="BlockMask.BlockMaskLoc"/> and <see cref="BlockMask.BlockMaskExt"/>
/// </summary>
/// <param name="SBS"></param>
public BlockMaskBase(SubBlockSelector SBS, MPI_Comm MPIcomm)
{
m_map = SBS.GetMapping;
m_sbs = SBS;
m_AggBS = m_map.AggBasis;
m_DGdegree = m_map.DgDegree;
m_Ni0 = Ni0Gen();
m_NoOfVariables = m_AggBS.Length;
//Testen ob es cells gibt wo Var<>NoOfVar, das würde dementsprechend auch m_DG beeinflussen
m_NoOfSpecies = new int[m_NoOfCells][];
for (int iCell = 0; iCell < m_NoOfCells; iCell++)
{
m_NoOfSpecies[iCell] = new int[m_NoOfVariables];
for (int iVar = 0; iVar < m_NoOfVariables; iVar++)
{
m_NoOfSpecies[iCell][iVar] = m_AggBS[iVar].GetNoOfSpecies(iCell + m_CellOffset);
}
}
}
/// <summary>
/// ctor
/// </summary>
public MPIEnviroment()
{
m_mpi_comm = csMPI.Raw._COMM.WORLD;
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out m_MPISize);
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out m_MPI_Rank);
m_hostname = Dns.GetHostName();
IPHostEntry e = Dns.GetHostEntry(m_hostname);
if (e != null)
{
string hostname = e.HostName;
if (hostname != null)
{
m_hostname = hostname;
}
}
m_hostname = m_hostname.ToLowerInvariant();
m_HostnameForRank = m_hostname.MPIAllGatherO(csMPI.Raw._COMM.WORLD);
{
m_AllHostNames = new Dictionary <string, int[]>();
for (int iRnk = 0; iRnk < MPI_Size; iRnk++)
{
var hn = m_HostnameForRank[iRnk];
int[] Ranks;
if (!m_AllHostNames.TryGetValue(hn, out Ranks))
{
Ranks = new int[0];
m_AllHostNames.Add(hn, Ranks);
}
iRnk.AddToArray(ref Ranks);
m_AllHostNames[hn] = Ranks;
}
}
SMPEvaluation();
}
/// <summary>
/// see ParMETIS manual;
/// </summary>
public static METIS.ReturnCodes V3_RefineKway(
int[] vtxdist,
int[] xadj,
int[] adjncy,
int[] vwgt,
int[] adjwgt,
ref int wgtflag,
ref int numflag,
ref int ncon,
ref int nparts,
float[] tpwgts,
float[] ubvec,
int[] options,
ref int edgecut,
int[] part,
MPI_Comm MPI_Comm)
{
return((METIS.ReturnCodes)ParMETIS_V3_RefineKway(
vtxdist, xadj, adjncy, vwgt, adjwgt, ref wgtflag, ref numflag, ref ncon, ref nparts, tpwgts, ubvec, options, ref edgecut, part, csMPI.Raw._COMM.Comm_f2c(MPI_Comm)));
}
internal override object Get(MPI_Comm comm)
{
IntPtr?ptr = GetIntPtr(comm);
if (ptr == null)
{
return(null);
}
if (onHeap)
{
return(Marshal.PtrToStructure(ptr.Value, typeof(T)));
}
else
{
IntPtr ptrValue = ptr.Value;
unsafe
{
return(Marshal.PtrToStructure(new IntPtr(&ptrValue), typeof(T)));
}
}
}
/// <summary>
/// Constructs a new mapping from an ordered list of basis functions;
/// </summary>
/// <param name="_basis">the list of DG basis'es that define this mapping</param>
private UnsetteledCoordinateMapping(IGridData g, IEnumerable <Basis> _basis, MPI_Comm _Comm) :
base(ComputeLength(_basis), _Comm) //
{
// =========================
// initial checks
// =========================
Basis[] basis = _basis.ToArray();
m_Context = g;
for (int i = 0; i < basis.Length; i++)
{
if (!object.ReferenceEquals(m_Context, basis[i].GridDat))
{
throw new ArgumentException("all basis-objects must be associated with the same grid.");
}
}
m_BasisS = basis;
// cell - independent index calculation
//int[] j0CoordinateIndexMin = new int[m_BasisS.Length + 1];
//int[] j0CoordinateIndexMax = new int[m_BasisS.Length + 1];
m_j0CoordinateIndex = new int[m_BasisS.Length + 1];
m_MaxTotalNoOfCoordinatesPerCell = 0;
m_MinTotalNoOfCoordinatesPerCell = 0;
for (int i = 0; i < m_BasisS.Length; i++)
{
m_j0CoordinateIndex[i] = m_MaxTotalNoOfCoordinatesPerCell;
m_MaxTotalNoOfCoordinatesPerCell += m_BasisS[i].MaximalLength;
m_MinTotalNoOfCoordinatesPerCell += m_BasisS[i].MinimalLength;
}
m_j0CoordinateIndex[m_BasisS.Length] = m_MaxTotalNoOfCoordinatesPerCell;
// ==================
// sub-blocking setup
// ==================
m_SubblkLen = new List <int[]>();
}
/// <summary>
/// see ParMETIS manual;
/// </summary>
public static METIS.ReturnCodes V3_PartGeomKway(
int[] vtxdist,
int[] xadj,
int[] adjncy,
int[] vwgt,
int[] adjwgt,
int[] wgtflag,
ref int numflag,
ref int ndims,
float[] xyz,
ref int ncon,
ref int nparts,
float[] tpwgts,
float[] ubvec,
int[] options,
int[] edgecut,
int[] part,
ref MPI_Comm MPI_Comm)
{
return((METIS.ReturnCodes)V3_PartGeomKway(
vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, ref numflag, ref ndims, xyz, ref ncon, ref nparts, tpwgts, ubvec, options, edgecut, part, csMPI.Raw._COMM.Comm_f2c(MPI_Comm)));
}
/// <summary>
/// Retrieve the value of this attribute in a communicator,
/// if it exists.
/// </summary>
/// <param name="comm">The communicator to query.</param>
/// <returns>The value of this attribute in the communicator, if any.</returns>
internal IntPtr?GetIntPtr(MPI_Comm comm)
{
int flag;
IntPtr result;
unsafe
{
int errorCode = Unsafe.MPI_Attr_get(comm, keyval, new IntPtr(&result), out flag);
if (errorCode != Unsafe.MPI_SUCCESS)
{
throw Environment.TranslateErrorIntoException(errorCode);
}
}
if (flag != 0)
{
return(result);
}
else
{
return(null);
}
}
/// <summary>
/// ctor
/// </summary>
public MPIEnviroment()
{
m_mpi_comm = csMPI.Raw._COMM.WORLD;
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out m_MPISize);
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out m_MPI_Rank);
//if (m_MPI_Rank == 1)
// Debugger.Break();
m_hostname = Dns.GetHostName();
IPHostEntry e = Dns.GetHostEntry(m_hostname);
if (e != null)
{
string hostname = e.HostName;
if (hostname != null)
{
m_hostname = hostname;
}
}
m_hostname = m_hostname.ToLowerInvariant();
SMPEvaluation();
}
/// <summary> /// Set this value's attribute to a particular value. /// </summary> /// <param name="comm">The communicator to modify.</param> /// <param name="value">The value to store.</param> internal abstract void Set(MPI_Comm comm, Object value);
/// <summary> /// Remove this attribute from a particular communicator. /// </summary> /// <param name="comm">The communicator storing the attribute.</param> internal abstract void Remove(MPI_Comm comm);
public static unsafe extern int MPI_Comm_rank(MPI_Comm comm, out int rank);
public static unsafe extern int MPI_Cart_create(MPI_Comm comm, int ndims, int* dims, int* periods, int reorder, MPI_Comm* newcomm);
public static unsafe extern int MPI_Comm_create(MPI_Comm comm, MPI_Group group, out MPI_Comm newcomm);
public static unsafe extern int MPI_Scatterv(IntPtr sendbuf, int[] sendcounts, int[] displs, MPI_Datatype sendtype,
IntPtr recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm);
public static extern unsafe int MPI_Attr_delete(MPI_Comm comm, int keyval);
public static unsafe extern int MPI_Pack_size(int incount, MPI_Datatype datatype, MPI_Comm comm, out int size);
public static extern unsafe int MPI_Attr_put(MPI_Comm comm, int keyval, IntPtr attribute_val);
public static extern unsafe int MPI_Attr_get(MPI_Comm comm, int keyval, IntPtr attribute_val, out int flag);
public static unsafe extern int MPI_Comm_free(ref MPI_Comm comm);
public static unsafe extern int MPI_Graph_create(MPI_Comm comm, int nnodes, int* index, int* edges, int reorder, MPI_Comm *newComm);
public static unsafe extern int MPI_Cart_sub(MPI_Comm comm, int* remain_dims, MPI_Comm* newcomm);
/// <summary>
/// Does not copy the attribute. This is the C# equivalent of
/// MPI's <c>MPI_NULL_COPY_FN</c>, used to work around a Mono
/// crash in AttributesTest.
/// </summary>
private int NullCopy(MPI_Comm comm, int keyval, IntPtr extra_state, IntPtr attribute_val_in,
IntPtr attribute_val_out, out int flag)
{
flag = 0;
return(Unsafe.MPI_SUCCESS);
}
public static unsafe extern int MPI_Comm_test_inter(MPI_Comm comm, out int flag);
public static unsafe extern int MPI_Reduce_scatter(IntPtr sendbuf, IntPtr recvbuf, int[] recvcounts, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
public static unsafe extern int MPI_Comm_remote_size(MPI_Comm comm, out int size);
/// <summary>
/// Exchanges serializeable data objects in between all MPI processes.
/// </summary>
/// <param name="objects_to_send">
/// Data to send.<br/>
/// keys: MPI rank of the process to which <em>o</em> should be send to<br/>
/// values: some object <em>o</em>
/// </param>
/// <param name="comm"></param>
/// <returns>
/// Received data.<br/>
/// keys: MPI rank of the process from which <em>q</em> has been received.<br/>
/// values: some object <em>q</em>
/// </returns>
public static IDictionary <int, T> ExchangeData <T>(IDictionary <int, T> objects_to_send, MPI_Comm comm)
{
using (var sms = new SerialisationMessenger(comm)) {
sms.SetCommPathsAndCommit(objects_to_send.Keys);
foreach (var kv in objects_to_send)
{
sms.Transmitt(kv.Key, kv.Value);
}
var R = new Dictionary <int, T>();
T obj;
int rcv_rank;
while (sms.GetNext(out rcv_rank, out obj))
{
R.Add(rcv_rank, obj);
}
return(R);
}
}
public static unsafe extern int MPI_Comm_remote_group(MPI_Comm comm, out MPI_Group group);
public static unsafe extern int MPI_Comm_compare(MPI_Comm comm1, MPI_Comm comm2, out int result);
public int HYPRE_ParCSRPCGCreate(MPI_Comm com, out HYPRE_Solver solver)
{
return(ParCSRPCGCreate(com, out solver));
}
public static unsafe extern int MPI_Scan(IntPtr sendbuf, IntPtr recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
private void ConstructorCommon(int FrameBlockSize, int[][] _Subblk_i0, int[][] _SubblkLen, int[] _BlockType, MPI_Comm MpiComm)
{
MPICollectiveWatchDog.Watch(MpiComm);
int LocalLength = base.LocalLength;
// ===============
// check arguments
// ===============
if (_Subblk_i0.Length != _SubblkLen.Length)
{
throw new ArgumentException();
}
for (int i = _Subblk_i0.Length - 1; i >= 0; i--)
{
if (_Subblk_i0[i].Length != _SubblkLen[i].Length)
{
throw new ArgumentException();
}
int i0min = 0;
for (int iSblk = 0; iSblk < _Subblk_i0[i].Length; iSblk++)
{
if (_Subblk_i0[i][iSblk] < i0min)
{
throw new ArgumentException("Potential sub-block overlapping.");
}
i0min += _SubblkLen[i][iSblk];
}
}
this.m_Subblk_i0 = _Subblk_i0;
this.m_SubblkLen = _SubblkLen;
if (FrameBlockSize >= 1)
{
for (int BlockType = 0; BlockType < _Subblk_i0.Length; BlockType++)
{
int NoOfSubblocks = _Subblk_i0[BlockType].Length;
int BlockLen = _Subblk_i0[BlockType][NoOfSubblocks - 1] + _SubblkLen[BlockType][NoOfSubblocks - 1];
if (BlockLen > FrameBlockSize)
{
throw new ArgumentException();
}
}
if (LocalLength % FrameBlockSize != 0)
{
throw new ArgumentException("'FrameBlockSize', if specified, must be a divider of 'LocalLength'.");
}
if (_BlockType != null)
{
if (_BlockType.Length != LocalLength / FrameBlockSize)
{
throw new ArgumentException("Mismatch between number of blocks specified by '_BlockType' and 'LocalLength/FrameBlockSize'.");
}
}
}
#if DEBUG
if (_BlockType != null)
{
for (int j = 0; j < _BlockType.Length; j++)
{
if (_BlockType[j] < 0 || _BlockType[j] > _Subblk_i0.Length)
{
throw new ArgumentException();
}
}
}
#endif
// ====================
// set internal members
// ====================
int LocalNoOfBlocks;
if (_BlockType != null)
{
LocalNoOfBlocks = _BlockType.Length;
if (FrameBlockSize > 0)
{
//throw new ArgumentException("If no BlockTape is given, the FrameBlockSize must be specified.");
if (LocalLength / FrameBlockSize != _BlockType.Length)
{
throw new ArgumentException("FrameBlockSize does not match _BlockType.");
}
}
}
else
{
if (FrameBlockSize <= 0)
{
throw new ArgumentException("If no BlockTape is given, the FrameBlockSize must be specified.");
}
LocalNoOfBlocks = LocalLength / FrameBlockSize;
}
m_BlocksPartition = new Partitioning(LocalNoOfBlocks, MpiComm);
if (_Subblk_i0.Length > 1)
{
this.m_BlockType = _BlockType;
}
else
{
this.m_BlockType = null;
}
int J = _BlockType.Length;
#if DEBUG
{
var fbMin = FrameBlockSize.MPIMin(MpiComm);
var fbMax = FrameBlockSize.MPIMax(MpiComm);
if (fbMin != FrameBlockSize || fbMin != FrameBlockSize)
{
throw new ApplicationException("MPI bug: different FrameBlockSize among processors.");
}
}
#endif
if (FrameBlockSize == 0)
{
throw new ArgumentException();
}
else if (FrameBlockSize < 0)
{
int[] BlockI0 = new int[J + 1];
BlockI0[0] = base.i0;
int FrameBlockSizeMaybe = -1;
for (int j = 0; j < J; j++)
{
int bT = _BlockType[j];
int B = _Subblk_i0[bT].Length;
int BlockLen = _Subblk_i0[bT][B - 1] + _SubblkLen[bT][B - 1];
BlockI0[j + 1] = BlockI0[j] + BlockLen;
if (j == 0)
{
FrameBlockSizeMaybe = BlockLen;
}
else
{
if (FrameBlockSizeMaybe != BlockLen)
{
FrameBlockSizeMaybe = -1;
}
}
}
Debug.Assert(BlockI0[J] == base.LocalLength + base.i0);
int FrameBlockSizeMaybeMin = FrameBlockSizeMaybe.MPIMin(MpiComm);
int FrameBlockSizeMaybeMax = FrameBlockSizeMaybe.MPIMax(MpiComm);
if (FrameBlockSizeMaybeMin > 0 && FrameBlockSizeMaybe == FrameBlockSizeMaybeMin && FrameBlockSizeMaybe == FrameBlockSizeMaybeMax)
{
// constant blocking, although the stupid caller told us different
m_FrameBlockSize = FrameBlockSizeMaybe;
m_Block_i0 = null;
}
else
{
m_Block_i0 = BlockI0;
m_FrameBlockSize = -1;
}
}
else
{
Debug.Assert(FrameBlockSize >= 1);
m_FrameBlockSize = FrameBlockSize;
}
}
public static unsafe extern int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm bridge_comm, int remote_leader, int tag, out MPI_Comm newintercomm);
public static unsafe extern int MPI_Comm_split(MPI_Comm comm, int color, int key, out MPI_Comm newcomm);
extern private static int Create(MPI_Comm comm, int ilower, int iupper, int jlower, int jupper, out HYPRE_IJMatrix matrix);
public static unsafe extern int MPI_Intercomm_merge(MPI_Comm intercomm, int high, out MPI_Comm newintracomm);
public BlockPartitioning(int LocalLength, IEnumerable <int> BlockI0, IEnumerable <int> BlockLen, MPI_Comm MpiComm, bool i0isLocal = false) : base(LocalLength, MpiComm)
{
//var enu_I0 = BlockI0.GetEnumerator();
//var enuLen = BlockLen.GetEnumerator();
Debug.Assert(base.LocalLength == LocalLength);
List <int> _SubblkLen1 = new List <int>();
List <int> _SubblkLen2 = new List <int>();
List <int> _BlockType = new List <int>();
//int NoOfBlocks = 0;
//while(enu_I0.MoveNext()) {
// if (!enuLen.MoveNext())
// throw new ArgumentException("Both enumerations must contain the same number of elements.");
// int N = enuLen.Current;
// int i0Block = enu_I0.Current;
// if (i0isLocal)
// i0Block += base.i0;
// base.IsInLocalRange(i0Block);
// base.IsInLocalRange(Math.Max(i0Block, i0Block + N - 1));
// NoOfBlocks++;
//}
//if (enuLen.MoveNext())
// throw new ArgumentException("Both enumerations must contain the same number of elements.");
int[] _BlockI0 = BlockI0.ToArray();
int[] _BlockLen = BlockLen.ToArray();
if (_BlockI0.Length != _BlockLen.Length)
{
throw new ArgumentException("Both enumerations must contain the same number of elements.");
}
int NoOfBlocks = _BlockI0.Length;
int[] BlockType = new int[NoOfBlocks];
int FrameBlockSize = -1;
for (int iBlock = 0; iBlock < NoOfBlocks; iBlock++)
{
int i0Block = _BlockI0[iBlock];
if (i0isLocal)
{
i0Block += base.i0;
}
int N = _BlockLen[iBlock];
if (!base.IsInLocalRange(i0Block))
{
throw new ArgumentException("Block i0 out of local range");
}
if (!base.IsInLocalRange(Math.Max(i0Block, i0Block + N - 1)))
{
throw new ArgumentException("Block end out of local range");
}
int iEBlock;
if (iBlock < NoOfBlocks - 1)
{
iEBlock = _BlockI0[iBlock + 1];
if (i0isLocal)
{
iEBlock += base.i0;
}
}
else
{
iEBlock = LocalLength + base.i0;
}
if (i0Block + N > iEBlock)
{
throw new ArgumentException("Block Length exceeds i0 of next block.");
}
int NE = iEBlock - i0Block;
if (iBlock == 0)
{
FrameBlockSize = NE;
}
else
{
if (NE != FrameBlockSize)
{
FrameBlockSize = -1;
}
}
int iBlockType;
bool bFound = false;
for (iBlockType = 0; iBlockType < _SubblkLen1.Count; iBlockType++)
{
if (N == _SubblkLen1[iBlockType] && NE == _SubblkLen2[iBlockType])
{
bFound = true;
break;
}
}
Debug.Assert(bFound == (iBlockType < _SubblkLen1.Count));
if (!bFound)
{
_SubblkLen1.Add(N);
_SubblkLen2.Add(NE);
iBlockType = _SubblkLen1.Count - 1;
}
Debug.Assert(N == _SubblkLen1[iBlockType]);
Debug.Assert(NE == _SubblkLen2[iBlockType]);
BlockType[iBlock] = iBlockType;
}
//
Debug.Assert(_SubblkLen1.Count == _SubblkLen2.Count);
int NoOfBlockTypes = _SubblkLen1.Count;
int[][] i0_Sblk = NoOfBlockTypes.ForLoop(iBlkType => new int[] { 0 });
int[][] LenSblk = NoOfBlockTypes.ForLoop(iBlkType => new int[] { _SubblkLen1[iBlkType] });
ConstructorCommon(FrameBlockSize, i0_Sblk, LenSblk, BlockType, MpiComm);
}
public static unsafe extern int MPI_Cartdim_get(MPI_Comm comm, int* result);
/// <summary> /// Retrieve this attribute's value from a particular communicator. /// </summary> /// <param name="comm">The communicator to query.</param> /// <returns>The object stored in the communicator, if any.</returns> internal abstract Object Get(MPI_Comm comm);
public static unsafe extern int MPI_Cart_get(MPI_Comm comm, int ndims, int* dims, int* periods, int* coords);
public static unsafe extern int MPI_Bcast(IntPtr buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm);
public static extern unsafe int MPI_Comm_get_parent(out MPI_Comm comm);
/// <summary>
/// Does not delete the attribute. This is the C# equivalent
/// of MPI's <c>MPI_NULL_DELETE_FN</c>, used to work around a
/// Mono crash in AttributesTest.
/// </summary>
private int NullDelete(MPI_Comm comm, int keyval, IntPtr attribute_val, IntPtr extra_state)
{
return(Unsafe.MPI_SUCCESS);
}
public static unsafe extern int MPI_Gather(IntPtr sendbuf, int sendcount, MPI_Datatype sendtype,
IntPtr recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm);
/// <summary>
/// Delete the memory associated with an attribute allocated on the heap.
/// Use as a parameter to <see cref="Unsafe.MPI_Keyval_create"/>
/// </summary>
private int DeleteAttributeMemory(MPI_Comm comm, int keyval, IntPtr attribute_val, IntPtr extra_state)
{
Marshal.FreeHGlobal(attribute_val);
return(Unsafe.MPI_SUCCESS);
}
/// <summary>
/// MPI-parallel 2-norm
/// </summary>
static public double MPI_L2Norm <V>(this V vec, MPI_Comm comm) where V : IList <double>
{
return(drnm2(vec.Count, vec, 1, comm));
}
internal AttributeSet(MPI_Comm comm)
{
this.comm = comm;
this.objectAttributes = new Dictionary <Attribute, object>();
}
public static unsafe extern int MPI_Unpack(IntPtr inbuf, int insize, ref int position, IntPtr outbuf, int outcount, MPI_Datatype datatype, MPI_Comm comm);
/// <summary>
/// Constructs a block partitioning from a sub-vector index list.
/// </summary>
/// <param name="B"></param>
/// <param name="SubvectorIdx">
/// Indices into <paramref name="B"/> that should be in the result.
/// </param>
/// <param name="comm"></param>
/// <param name="FrameBlockSize">
/// <see cref="IBlockPartitioning.AllBlockSizesEqual"/>;
/// The size of the frame block:
/// - if negative, not used; in this case, the length of each block can be different and is defined by the sub-blocking, i.e. by <paramref name="_Subblk_i0"/>, <paramref name="_SubblkLen"/> and <paramref name="_BlockType"/>. In this case <see cref="AllBlockSizesEqual"/> is false.
/// - if positive, a frame of constant size is assumed around each block, i.e. the index range of the i-th block starts at index i*<paramref name="FrameBlockSize"/>; then <see cref="AllBlockSizesEqual"/> is true. However, the sub-blocking can still differ from block to block.
/// </param>
/// <returns></returns>
public static BlockPartitioning GetSubBlocking <T>(this IBlockPartitioning B, T SubvectorIdx, MPI_Comm comm, int FrameBlockSize = -1)
where T : IEnumerable <int>
{
List <int[]> Subblk_i0 = new List <int[]>();
List <int[]> SubblkLen = new List <int[]>();
List <int> BlockType = new List <int>();
int iCheck = -1;
int iCurrentBlock = -1, CurrentBlockType = -12, B_i0 = -1, BLen = -1;
int[] B_sblk_i0 = null;
int[] B_sblkLen = null;
bool[] SblkMarker = null;
int LocalLength = 0;
List <int> tmp_i0 = new List <int>();
List <int> tmpLen = new List <int>();
int FrameBlkAuto = 0;
foreach (int i in SubvectorIdx)
{
B.TestIfInLocalRange(i);
if (i < iCheck)
{
throw new ArgumentException("Sub-vector indices must be sorted.");
}
iCheck = i;
int iBlock = B.GetBlockIndex(i);
if (iBlock != iCurrentBlock)
{
// block change - need to do something
// record actual block
if (iCurrentBlock >= 0)
{
RecordBlock(Subblk_i0, SubblkLen, BlockType, B_sblk_i0, B_sblkLen, SblkMarker, tmp_i0, tmpLen, ref FrameBlkAuto);
}
// move on to next block
iCurrentBlock = iBlock;
CurrentBlockType = B.GetBlockType(iBlock);
B_sblk_i0 = B.GetSubblk_i0(CurrentBlockType);
B_sblkLen = B.GetSubblkLen(CurrentBlockType);
B_i0 = B.GetBlockI0(iBlock);
BLen = B.GetBlockLen(BLen);
if (SblkMarker == null || SblkMarker.Length < BLen)
{
SblkMarker = new bool[BLen];
}
else
{
Array.Clear(SblkMarker, 0, SblkMarker.Length);
}
}
int iWithin = i - B_i0;
Debug.Assert(iWithin >= 0);
Debug.Assert(iWithin < BLen);
SblkMarker[iWithin] = true;
LocalLength++;
}
// record actual block
if (iCurrentBlock >= 0)
{
RecordBlock(Subblk_i0, SubblkLen, BlockType, B_sblk_i0, B_sblkLen, SblkMarker, tmp_i0, tmpLen, ref FrameBlkAuto);
}
Debug.Assert(LocalLength == SubvectorIdx.Count());
// Debug.Assert(BlockType.Count == B.LocalNoOfBlocks);
// finalize data structure
// -----------------------
if (FrameBlockSize == 0)
{
FrameBlockSize = FrameBlkAuto;
}
if (FrameBlockSize > 0)
{
LocalLength = FrameBlockSize * BlockType.Count;
}
return(new BlockPartitioning(LocalLength, FrameBlockSize,
Subblk_i0.ToArray(), SubblkLen.ToArray(), BlockType.ToArray(), comm));
}
public static unsafe extern int MPI_Comm_dup(MPI_Comm comm, out MPI_Comm comm_out);
/// <summary>
/// Exchanges serialize-able data objects in between all MPI processes.
/// </summary>
/// <param name="objects_to_send">
/// Data to send.<br/>
/// keys: MPI rank of the process to which <em>o</em> should be send to<br/>
/// values: some object <em>o</em>
/// </param>
/// <param name="comm"></param>
/// <returns>
/// Received data.<br/>
/// keys: MPI rank of the process from which <em>q</em> has been received.<br/>
/// values: some object <em>q</em>
/// </returns>
public static IDictionary <int, T> ExchangeData <T>(IDictionary <int, T> objects_to_send, MPI_Comm comm)
{
using (var sms = new SerialisationMessenger(comm)) {
//if (PoorManDebugger != null) {
// PoorManDebugger.WriteLine("tag offset is " + sms.m_MyTagOffset);
// PoorManDebugger.Flush();
//}
sms.SetCommPathsAndCommit(objects_to_send.Keys);
foreach (var kv in objects_to_send)
{
sms.Transmit(kv.Key, kv.Value);
}
var R = new Dictionary <int, T>();
T obj;
int rcv_rank;
while (sms.GetNext(out rcv_rank, out obj))
{
R.Add(rcv_rank, obj);
}
//if(PoorManDebugger != null)
//{
// PoorManDebugger.WriteLine(" leaving ExchangeData");
// PoorManDebugger.Flush();
//}
return(R);
}
}
public static extern unsafe int MPI_Comm_spawn(byte* command, byte** argv, int maxprocs, MPI_Info info, int root, MPI_Comm comm, out MPI_Comm intercomm, int* array_of_errorcodes);
public static extern int MPI_Barrier(MPI_Comm comm);
public static extern unsafe int MPI_Comm_spawn_multiple(int count, ref byte* array_of_commands, ref byte** array_of_argv, ref int array_of_maxprocs, ref MPI_Info array_of_info, int root, MPI_Comm comm, out MPI_Comm intercomm, out int array_of_errorcodes);
