/// <summary>
/// Creates the catchment communicators.
/// </summary>
/// <param name="workAllocator">The work allocator.</param>
private void CreateCommunicators(IWorkAllocator workAllocator)
{
// create the communicators (creating communicators is a collective operation, all processes that belong to the new communicator
// must participate in the call).
// It is also important that all processes create the communicators in the same order to avoid deadlock
// There is one communicator per catchment
Log.DebugFormat("Rank {0}: Creating communicators", WorldRank);
// we need to sort the list of catchments by catchment CatchmentId.
// All processes must have the same catchment order or they can deadlock when creating communicators that span processes.
GlobalDefinition.SortCatchmentsById();
foreach (CatchmentDefinition catchment in GlobalDefinition)
{
Log.DebugFormat("Rank {0}: catchment {1} Creating communicator for {2} processes", WorldRank, catchment.Id, workAllocator.RanksByCatchment[catchment.Id].Count);
IGroupProxy catchmentGroup = CreateGroup(workAllocator.RanksByCatchment[catchment.Id].ToArray());
Log.DebugFormat("Rank {0}: Catchment group created, size = {1}", WorldRank, catchmentGroup.Size);
IIntracommunicatorProxy catchmentCommunicator = CreateIntracommunicatorProxy(catchmentGroup);
if (catchmentCommunicator != null)
{
Log.DebugFormat("Rank {0}: Communicator created, rank = {1} size = {2}", WorldRank, catchmentCommunicator.GetRank(this.WorldRank), catchmentCommunicator.Size);
}
else
{
Log.DebugFormat("Rank {0}: Communicator created, I am not a member", WorldRank);
}
// catchmentCommunicator will be null if the current rank is not a member of the catchmentGroup.
// This is OK, as each rank only requires the communicators for catchments it is involved in.
if (catchmentCommunicator != null)
{
Debug.Assert(workAllocator.RanksByCatchment[catchment.Id].Contains(WorldRank));
communicatorsByCatchmentId.Add(catchment.Id, catchmentCommunicator);
#if CELL_WEIGHTED_SUMS
// If I am the catchment coordinator for at least one catchment, then I will need the dictionary of cached
// catchment statistics evaluators
if (catchmentCommunicator.GetRank(this.WorldRank) == 0)
{
Log.DebugFormat("Rank {0}: I am catchment coordinator. Creating stats evaluator cache.", WorldRank);
// count how often this process acts as catchment coordinator
CatchmentCoordinatorCount++;
// create the statistics evaluator cache
if (CatchmentStatisticsEvaluatorCache == null)
{
CatchmentStatisticsEvaluatorCache = new Dictionary <string, CatchmentStatisticsEvaluator <ICloneableSimulation, MpiSysConfig> >();
}
}
#endif
}
else
{
Debug.Assert(!workAllocator.RanksByCatchment[catchment.Id].Contains(WorldRank));
}
}
}
private void DebugDumpWorkAllocator(IWorkAllocator workAllocator)
{
#if DEBUG_MODELS
Log.DebugFormat("Rank {0}: gridded result count = {1}", WorldRank, workAllocator.GriddedResultCount);
for (int i = 0; i < GetWorldSize(); i++)
{
Log.DebugFormat("Rank {0} sees {2} catchment results for rank {1}", WorldRank, i, workAllocator.NumCatchmentResultsPerWorker[i]);
Log.DebugFormat("Rank {0} sees {2} gridded results for rank {1}", WorldRank, i, workAllocator.NumGriddedResultsPerWorker[i]);
}
foreach (CatchmentDefinition catchment in GlobalDefinition)
{
HashSet <int> ranks = workAllocator.RanksByCatchment[catchment.Id];
string msg = String.Format("Rank {0}: Catchment {1} has {2} ranks: ", WorldRank, catchment.Id, ranks.Count);
foreach (int rank in ranks)
{
msg += String.Format("{0}, ", rank);
}
Log.DebugFormat(msg);
}
#endif
}
/// <summary>
/// Allocates work and distributes the work packages to all workers.
/// </summary>
/// <param name="workAllocator">The work allocator. Different allocation strategies can be selected through this argument.</param>
private void AllocateWork(IWorkAllocator workAllocator)
{
if (workAllocator == null)
{
throw new ArgumentNullException("workAllocator");
}
Log.DebugFormat("Rank {0}: Allocating work", WorldRank);
workAllocator.Allocate();
DebugDumpWorkAllocator(workAllocator);
// create the MPI communicators used to coordinate processes involved in each catchment
CreateCommunicators(workAllocator);
// we only need to preserve some data from the work allocator
SetWorkPackage(workAllocator.WorkPackage);
NumCatchmentResultsPerWorker = workAllocator.NumCatchmentResultsPerWorker;
NumGriddedResultsPerWorker = workAllocator.NumGriddedResultsPerWorker;
TotalCellCount = workAllocator.GriddedResultCount;
// check that we have enough work for the number of workers
if (IsMaster && TotalCellCount < GetWorldSize() - 1)
{
throw new ConfigurationException(
String.Format(
"The number of worker processes cannot be greater than the number of cells. Currently: {0} workers, {1} cells",
GetWorldSize() - 1,
TotalCellCount));
}
#if !CELL_WEIGHTED_SUMS
CalculateCoordinatorCount();
#endif
Log.DebugFormat("Rank {0}: AllocateWork complete", WorldRank);
}
