public static void SetupParallelism(ref string[] args)
{
// Set up MPI
SALSAUtility.MPI_Environment = new Environment(ref args);
SALSAUtility.MPI_communicator = Communicator.world; //initializing MPI world communicator
SALSAUtility.MPI_Rank = SALSAUtility.MPI_communicator.Rank; // Rank of this process
SALSAUtility.MPI_Size = SALSAUtility.MPI_communicator.Size; // Number of MPI Processes
// Set up MPI
SALSAUtility.MPIperNodeCount = SALSAUtility.MPI_Size / SALSAUtility.NodeCount;
//smbeason
ManxcatCentral.Configuration.MPIperNodeCount = SALSAUtility.MPIperNodeCount;
//smbeason
if ((SALSAUtility.MPIperNodeCount * SALSAUtility.NodeCount) != SALSAUtility.MPI_Size)
{
Exception e = SALSAUtility.SALSAError("Inconsistent MPI counts Nodes "
+ SALSAUtility.NodeCount.ToString() + " Size " +
SALSAUtility.MPI_Size.ToString());
throw (e);
}
SALSAUtility.ParallelPattern = "Machine:" + Environment.ProcessorName + " " +
SALSAUtility.ThreadCount.ToString() + "x" +
SALSAUtility.MPIperNodeCount.ToString() + "x" +
SALSAUtility.NodeCount.ToString();
if (SALSAUtility.MPI_Rank == 0)
{
SALSAUtility.SALSAPrint(0, " Distance Data Type: " + typeof(TDistance));
SALSAUtility.SALSAPrint(0, SALSAUtility.ParallelPattern);
}
}
// Read Distance Data
public static void ReadDataFromFile(string fname)
{
if ((SALSAUtility.DebugPrintOption > 0) && (SALSAUtility.MPI_Rank == 0))
{
SALSAUtility.SALSAPrint(1, "Starting to read data: " +
" Distance Cut " + SALSAUtility.DistanceCut.ToString("F3"));
}
double countremoveddistances = 0.0;
double counttotaldistances = 0.0;
SALSAUtility.StoredDistanceOption = Math.Max(2, SALSAUtility.StoredDistanceOption);
if (SALSAUtility.PointCount_Global == SALSAUtility.NumberOriginalPoints)
{
SALSAUtility.DiskDistanceOption = Math.Max(2, SALSAUtility.DiskDistanceOption);
}
// Remove unsupported options
if (SALSAUtility.DiskDistanceOption == 3)
{
SALSAUtility.StoredDistanceOption = 3;
}
if (SALSAUtility.StoredDistanceOption == 3)
{
SALSAUtility.CalcFixedCrossFixed = false;
}
// Set sizes of matrix on disk
int rowcount = SALSAUtility.PointCount_Global;
int colcount = SALSAUtility.PointCount_Global;
if (SALSAUtility.DiskDistanceOption == 1)
{
rowcount = SALSAUtility.NumberOriginalPoints;
colcount = SALSAUtility.NumberOriginalPoints;
}
if (SALSAUtility.DiskDistanceOption == 3)
{
rowcount = SALSAUtility.NumberVariedPoints;
}
// MatrixTextReader reader = new MatrixTextReader(rowcount,colcount);
var reader = new MatrixBinaryReader(rowcount, colcount);
bool Oneread = true;
if (SALSAUtility.StoredDistanceOption != SALSAUtility.DiskDistanceOption)
{
Oneread = false;
}
if (SALSAUtility.Usedreordered)
{
Oneread = false;
}
if (Oneread)
{
#if USE_UINT16
SALSAUtility.PointDistances = reader.ReadUInt16(fname, SALSAUtility.PointStart_Process, SALSAUtility.PointCount_Process);
#elif USE_INT16
SALSAUtility.PointDistances = reader.ReadInt16(fname, SALSAUtility.PointStart_Process,
SALSAUtility.PointCount_Process);
#else
SALSAUtility.PointDistances = reader.ReadDouble(fname, SALSAUtility.PointStart_Process, SALSAUtility.PointCount_Process);
#endif
int numberofcolumns = SALSAUtility.PointCount_Global;
for (int GlobalPointIndex = SALSAUtility.PointStart_Process;
GlobalPointIndex < SALSAUtility.PointStart_Process + SALSAUtility.PointCount_Process;
GlobalPointIndex++)
{
int rowindex = GlobalPointIndex;
if (SALSAUtility.StoredDistanceOption == 2)
{
rowindex = rowindex - SALSAUtility.PointStart_Process;
}
if (SALSAUtility.StoredDistanceOption == 3)
{
int originalpoint = SALSAUtility.UsedPointtoOriginalPointMap[rowindex];
int variedpoint = SALSAUtility.OriginalPointDisposition[originalpoint] - SALSAUtility.SALSASHIFT;
if (variedpoint < 0)
{
Exception e =
SALSAUtility.SALSAError(" Illegal Distance Request Used Point " + rowindex.ToString() +
" Original " + originalpoint.ToString());
throw (e);
}
rowindex = variedpoint - SALSAUtility.VariedPointStart_Process;
}
for (int columnindex = 0; columnindex < numberofcolumns; columnindex++)
{
TDistance Temp = SALSAUtility.PointDistances[rowindex][columnindex];
counttotaldistances = counttotaldistances + 1.0;
if (SALSAUtility.DistanceCut > 0.0)
{
double distancevalue = (SALSAUtility.PointDistances[rowindex][columnindex] /
(TDistance.MaxValue * 1.0));
if (distancevalue > SALSAUtility.DistanceCut)
{
SALSAUtility.PointDistances[rowindex][columnindex] = TDistance.MaxValue;
countremoveddistances = countremoveddistances + 1.0;
}
}
}
}
}
else
{
int colsread = SALSAUtility.PointCount_Global;
if (SALSAUtility.DiskDistanceOption == 1)
{
colsread = SALSAUtility.NumberOriginalPoints;
}
if (SALSAUtility.StoredDistanceOption == 2)
{
SALSAUtility.PointDistances = new TDistance[SALSAUtility.PointCount_Process][];
}
if (SALSAUtility.StoredDistanceOption == 3)
{
SALSAUtility.PointDistances = new TDistance[SALSAUtility.VariedPointCount_Process][];
}
for (int GlobalPointIndex = SALSAUtility.PointStart_Process;
GlobalPointIndex < SALSAUtility.PointStart_Process + SALSAUtility.PointCount_Process;
GlobalPointIndex++)
{
int rowtostore = GlobalPointIndex;
if (SALSAUtility.StoredDistanceOption == 2)
{
rowtostore = GlobalPointIndex - SALSAUtility.PointStart_Process;
}
if (SALSAUtility.StoredDistanceOption == 3)
{
int OriginalIndex = SALSAUtility.UsedPointtoOriginalPointMap[GlobalPointIndex];
rowtostore = SALSAUtility.OriginalPointDisposition[OriginalIndex] - SALSAUtility.SALSASHIFT;
if (rowtostore < 0)
{
continue;
}
rowtostore = rowtostore - SALSAUtility.VariedPointStart_Process;
}
int rowtoread = -1;
if (SALSAUtility.DiskDistanceOption == 1)
{
rowtoread = SALSAUtility.UsedPointtoOriginalPointMap[GlobalPointIndex];
}
if (SALSAUtility.DiskDistanceOption == 2)
{
rowtoread = SALSAUtility.ActualtoNaiveUsedOrder[GlobalPointIndex];
}
if (SALSAUtility.DiskDistanceOption == 3)
{
int OriginalIndex = SALSAUtility.UsedPointtoOriginalPointMap[GlobalPointIndex];
rowtoread = SALSAUtility.OriginalPointDisposition[OriginalIndex] - SALSAUtility.SALSASHIFT;
if (rowtoread < 0)
{
continue;
}
rowtoread = SALSAUtility.ActualtoNaiveUsedOrder[rowtoread];
}
SALSAUtility.PointDistances[rowtostore] = new TDistance[colcount];
#if USE_UINT16
buffer = reader.ReadUInt16(fname, rowtoread, 1);
#elif USE_INT16
buffer = reader.ReadInt16(fname, rowtoread, 1);
#else
buffer = reader.ReadDouble(fname, rowtoread, 1);
#endif
// Store buffer in PointDistances
for (int colIndex = 0; colIndex < colsread; colIndex++)
{
int coltostore = colIndex;
if (SALSAUtility.DiskDistanceOption == 1)
{
coltostore = SALSAUtility.OriginalPointtoUsedPointMap[colIndex];
if (coltostore < 0)
{
continue;
}
}
else if (SALSAUtility.DiskDistanceOption > 1)
{
coltostore = SALSAUtility.NaivetoActualUsedOrder[colIndex];
}
SALSAUtility.PointDistances[rowtostore][coltostore] = buffer[0][colIndex];
counttotaldistances = counttotaldistances + 1.0;
if (SALSAUtility.DistanceCut > 0.0)
{
double distancevalue = (SALSAUtility.PointDistances[rowtostore][coltostore] /
(TDistance.MaxValue * 1.0));
if (distancevalue > SALSAUtility.DistanceCut)
{
SALSAUtility.PointDistances[rowtostore][coltostore] = TDistance.MaxValue;
countremoveddistances = countremoveddistances + 1.0;
}
}
}
}
}
SALSAUtility.StartSubTimer(SALSAUtility.MPIREDUCETiming);
counttotaldistances = SALSAUtility.MPI_communicator.Allreduce(counttotaldistances, Operation <double> .Add);
countremoveddistances = SALSAUtility.MPI_communicator.Allreduce(countremoveddistances, Operation <double> .Add);
SALSAUtility.StopSubTimer(SALSAUtility.MPIREDUCETiming);
double fractionleft = 1.0 - countremoveddistances / counttotaldistances;
if ((SALSAUtility.DebugPrintOption > 0) && (SALSAUtility.MPI_Rank == 0))
{
SALSAUtility.SALSAPrint(1,
"Total Distances " + counttotaldistances.ToString("F0") +
" Distances Removed on Input " + countremoveddistances.ToString("F0") +
" Fraction Left " + fractionleft.ToString("F5"));
}
}