public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
}
Parallel.For(0, Program.ParallelOptions.MaxDegreeOfParallelism, Program.ParallelOptions, (ArrayThreadNo) =>
{
int beginindex = ParallelArrayRanges[ArrayThreadNo].StartIndex;
int indexlength = ParallelArrayRanges[ArrayThreadNo].Length;
for (int ArrayLoop = beginindex; ArrayLoop < beginindex + indexlength; ArrayLoop++)
{
TotalVectorSum[ArrayLoop] = 0.0;
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalVectorSum[ArrayLoop] += VectorSum[ThreadNo][ArrayLoop];
}
}
});
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
TotalVectorSum = PWCUtility.MPI_communicator.Allreduce <double>(TotalVectorSum, Operation <double> .Add);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
}
public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
for (int ArrayLoop = 0; ArrayLoop < ArraySize; ArrayLoop++)
{
Totalmean[ArrayLoop] += mean[ThreadNo][ArrayLoop];
}
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
Totalmean = PWCUtility.MPI_communicator.Allreduce <double>(Totalmean, Operation <double> .Add);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
if (TotalNumberofPoints < 0.5)
{
return;
}
for (int ArrayLoop = 0; ArrayLoop < ArraySize; ArrayLoop++)
{
Totalmean[ArrayLoop] = Totalmean[ArrayLoop] / TotalNumberofPoints;
}
}
public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
Totalmean += mean[ThreadNo];
Totalsquare += square[ThreadNo];
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
Totalmean = PWCUtility.MPI_communicator.Allreduce <double>(Totalmean, Operation <double> .Add);
Totalsquare = PWCUtility.MPI_communicator.Allreduce <double>(Totalsquare, Operation <double> .Add);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
if (TotalNumberofPoints < 0.5)
{
return;
}
Totalmean = Totalmean / TotalNumberofPoints;
Totalsquare = (Totalsquare / TotalNumberofPoints) - Totalmean * Totalmean;
Totalsigma = Math.Sqrt(Math.Max(0.0, Totalsquare));
}
public static void SetupParallelism(ref string[] args)
{
// Set up MPI
PWCUtility.MPI_Environment = new MPI.Environment(ref args);
PWCUtility.MPI_communicator = Communicator.world; //initializing MPI world communicator
PWCUtility.MPI_Rank = PWCUtility.MPI_communicator.Rank; // Rank of this process
PWCUtility.MPI_Size = PWCUtility.MPI_communicator.Size; // Number of MPI Processes
// Set up MPI
PWCUtility.MPIperNodeCount = PWCUtility.MPI_Size / PWCUtility.NodeCount;
if ((PWCUtility.MPIperNodeCount * PWCUtility.NodeCount) != PWCUtility.MPI_Size)
{
Exception e = PWCUtility.SALSAError("Inconsistent MPI counts Nodes "
+ PWCUtility.NodeCount.ToString() + " Size " + PWCUtility.MPI_Size.ToString());
throw (e);
}
PWCUtility.ParallelPattern = "Machine:" + MPI.Environment.ProcessorName.ToString() + " " + PWCUtility.ThreadCount.ToString() + "x" + PWCUtility.MPIperNodeCount.ToString() + "x" + PWCUtility.NodeCount.ToString();
if (PWCUtility.MPI_Rank == 0)
{
PWCUtility.SALSAPrint(0, " Distance Data Type: " + typeof(TDistance));
PWCUtility.SALSAPrint(0, PWCUtility.ParallelPattern);
}
} // End SetupParallelism
} // End synchronizeboolean(double cosmicdouble)
public static void SynchronizeMPIvariable(ref int cosmicint)
{
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIBROADCASTTiming);
PWCUtility.MPI_communicator.Broadcast <int>(ref cosmicint, 0);
PWCUtility.StopSubTimer(PWCUtility.MPIBROADCASTTiming);
}
return;
} // End synchronizeboolean(int cosmicint)
public void sumoverthreadsandmpi()
{
PWCUtility.StartSubTimer(PWCUtility.ThreadTiming);
Parallel.For(0, Program.ParallelOptions.MaxDegreeOfParallelism, Program.ParallelOptions, (ArrayThreadNo) =>
{
int beginindex = ParallelArrayRanges[ArrayThreadNo].StartIndex;
int indexlength = ParallelArrayRanges[ArrayThreadNo].Length;
for (int ArrayLoop = beginindex; ArrayLoop < beginindex + indexlength; ArrayLoop++)
{
double tmp = 0.0;
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
tmp += VectorSum[ThreadNo][ArrayLoop];
}
TotalVectorSum[ArrayLoop] = tmp;
}
});
PWCUtility.StopSubTimer(PWCUtility.ThreadTiming);
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
int bigsize = TotalVectorSum.Length;
if (bigsize <= 4096)
{
TotalVectorSum = PWCUtility.MPI_communicator.Allreduce <double>(TotalVectorSum, Operation <double> .Add);
}
else
{
double[] buffer = new double[4096];
int start = 0;
while (start < bigsize)
{
int whatsleft = Math.Min(bigsize - start, 4096);
for (int innerloop = 0; innerloop < whatsleft; innerloop++)
{
buffer[innerloop] = TotalVectorSum[start + innerloop];
}
buffer = PWCUtility.MPI_communicator.Allreduce <double>(buffer, Operation <double> .Add);
for (int innerloop = 0; innerloop < whatsleft; innerloop++)
{
TotalVectorSum[start + innerloop] = buffer[innerloop];
}
start += whatsleft;
}
}
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
}
public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
Total += TotalinThread[ThreadNo];
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
Total = PWCUtility.MPI_communicator.Allreduce <double>(Total, Operation <double> .Add);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
}
public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
TotalInt += Intvalue[ThreadNo];
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <int>(TotalNumberofPoints, Operation <int> .Add);
TotalInt = PWCUtility.MPI_communicator.Allreduce <int>(TotalInt, Operation <int> .Add);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
return;
}
public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
TotalOr = Orvalue[ThreadNo] || TotalOr;
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
TotalOr = PWCUtility.MPI_communicator.Allreduce <bool>(TotalOr, Operation <bool> .LogicalOr);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
return;
}
public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
TotalMax = Math.Max(TotalMax, Maxvalue[ThreadNo]);
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
TotalMax = PWCUtility.MPI_communicator.Allreduce <double>(TotalMax, Operation <double> .Max);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
return;
}
public static int OwnerforThisPoint(int GlobalPointIndex)
{ // Return process number for GlobalPointIndex
int startpoint = 0;
for (int mpiloop = 0; mpiloop < PWCUtility.MPI_Size; mpiloop++)
{
int endpoint = startpoint + PWCUtility.PointsperProcess[mpiloop];
if (GlobalPointIndex < endpoint)
{
return(mpiloop);
}
startpoint = endpoint;
}
PWCUtility.SALSAError(" Illegal Point in No Process " + GlobalPointIndex.ToString());
return(-1);
} // End InThisProcess(int GlobalPointIndex)
public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
for (int ArrayLoop = 0; ArrayLoop < ArraySize; ArrayLoop++)
{
TotalVectorSum[ArrayLoop] += VectorSum[ThreadNo][ArrayLoop];
}
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <int>(TotalNumberofPoints, Operation <int> .Add);
TotalVectorSum = PWCUtility.MPI_communicator.Allreduce <int>(TotalVectorSum, Operation <int> .Add);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
}
public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
if (IndexValue[ThreadNo] < 0)
{
continue;
}
TotalNumberofPoints += NumberofPoints[ThreadNo];
if (MinMaxPointer != 0)
{
if ((TotalIndexValue >= 0) && (TotalMaxOrMin > MaxOrMinvalue[ThreadNo]))
{
continue;
}
}
else
{
if ((TotalIndexValue >= 0) && (TotalMaxOrMin <= MaxOrMinvalue[ThreadNo]))
{
continue;
}
}
TotalMaxOrMin = MaxOrMinvalue[ThreadNo];
TotalIndexValue = IndexValue[ThreadNo];
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
if (MinMaxPointer != 0)
{
PWCUtility.AllReduceMaxWithIndex(ref TotalMaxOrMin, ref TotalIndexValue);
}
else
{
PWCUtility.AllReduceMinWithIndex(ref TotalMaxOrMin, ref TotalIndexValue);
}
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
return;
}
public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
for (int ArrayLoop = 0; ArrayLoop < ArraySize; ArrayLoop++)
{
TotalVectorMax[ArrayLoop] = Math.Max(TotalVectorMax[ArrayLoop], VectorMax[ThreadNo][ArrayLoop]);
}
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
TotalVectorMax = PWCUtility.MPI_communicator.Allreduce <double>(TotalVectorMax, Operation <double> .Max);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
}
public void sumoverthreadsandmpi()
{
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
for (int loop = 0; loop < NumberinSum; loop++)
{
TotalSum[loop] += Sum[ThreadNo][loop];
}
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
TotalSum = PWCUtility.MPI_communicator.Allreduce <double>(TotalSum, Operation <double> .Add);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
return;
}
public void sumoverthreadsandmpi()
{
for (int storeloop = 0; storeloop < Numbertofind; storeloop++)
{
TotalMinValue[storeloop] = -1.0;
TotalIndexValue[storeloop] = -1;
}
TotalWorst = -1;
for (int ThreadNo = 0; ThreadNo < NumberofThreads; ThreadNo++)
{
TotalNumberofPoints += NumberofPoints[ThreadNo];
for (int storeloop = 0; storeloop < Numbertofind; storeloop++)
{
if (IndexValuebythread[ThreadNo][storeloop] < 0)
{
continue; // End this thread
}
FindMinimumSet(MinValuebythread[ThreadNo][storeloop], IndexValuebythread[ThreadNo][storeloop], ref TotalWorst, TotalMinValue, TotalIndexValue, Numbertofind);
}
}
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
TotalNumberofPoints = PWCUtility.MPI_communicator.Allreduce <double>(TotalNumberofPoints, Operation <double> .Add);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
// Sort in absolute order and accumulate over processes. This takes Numbertofindsteps
for (int OrderLoop = 0; OrderLoop < Numbertofind; OrderLoop++)
{
int localindex = -1; // unset
double localvalue = -1.0;
int loopused = -1;
for (int internalloop = 0; internalloop < Numbertofind; internalloop++)
{ // Find minimum
if (TotalIndexValue[internalloop] < 0)
{
continue;
}
if ((localindex < 0) || (TotalMinValue[internalloop] < localvalue))
{
localindex = TotalIndexValue[internalloop];
localvalue = TotalMinValue[internalloop];
loopused = internalloop;
}
}
int oldlocalindex = localindex;
if (PWCUtility.MPI_Size > 1)
{
PWCUtility.StartSubTimer(PWCUtility.MPIREDUCETiming1);
PWCUtility.AllReduceMinWithIndex(ref localvalue, ref localindex);
PWCUtility.StopSubTimer(PWCUtility.MPIREDUCETiming1);
}
OrderedMinValue[OrderLoop] = localvalue;
OrderedIndexValue[OrderLoop] = localindex;
if ((oldlocalindex >= 0) && (OrderedIndexValue[OrderLoop] == oldlocalindex))
{
TotalIndexValue[loopused] = -1;
TotalMinValue[loopused] = -1.0;
}
} // Loop over Order Loop
return;
}