/// <summary>
/// Creates the catchment communicators.
/// </summary>
/// <param name="workAllocator">The work allocator.</param>
private void CreateCommunicators(IWorkAllocator workAllocator)
{
// create the communicators (creating communicators is a collective operation, all processes that belong to the new communicator
// must participate in the call).
// It is also important that all processes create the communicators in the same order to avoid deadlock
// There is one communicator per catchment
Log.DebugFormat("Rank {0}: Creating communicators", WorldRank);
// we need to sort the list of catchments by catchment CatchmentId.
// All processes must have the same catchment order or they can deadlock when creating communicators that span processes.
GlobalDefinition.SortCatchmentsById();
foreach (CatchmentDefinition catchment in GlobalDefinition)
{
Log.DebugFormat("Rank {0}: catchment {1} Creating communicator for {2} processes", WorldRank, catchment.Id, workAllocator.RanksByCatchment[catchment.Id].Count);
IGroupProxy catchmentGroup = CreateGroup(workAllocator.RanksByCatchment[catchment.Id].ToArray());
Log.DebugFormat("Rank {0}: Catchment group created, size = {1}", WorldRank, catchmentGroup.Size);
IIntracommunicatorProxy catchmentCommunicator = CreateIntracommunicatorProxy(catchmentGroup);
if (catchmentCommunicator != null)
{
Log.DebugFormat("Rank {0}: Communicator created, rank = {1} size = {2}", WorldRank, catchmentCommunicator.GetRank(this.WorldRank), catchmentCommunicator.Size);
}
else
{
Log.DebugFormat("Rank {0}: Communicator created, I am not a member", WorldRank);
}
// catchmentCommunicator will be null if the current rank is not a member of the catchmentGroup.
// This is OK, as each rank only requires the communicators for catchments it is involved in.
if (catchmentCommunicator != null)
{
Debug.Assert(workAllocator.RanksByCatchment[catchment.Id].Contains(WorldRank));
communicatorsByCatchmentId.Add(catchment.Id, catchmentCommunicator);
#if CELL_WEIGHTED_SUMS
// If I am the catchment coordinator for at least one catchment, then I will need the dictionary of cached
// catchment statistics evaluators
if (catchmentCommunicator.GetRank(this.WorldRank) == 0)
{
Log.DebugFormat("Rank {0}: I am catchment coordinator. Creating stats evaluator cache.", WorldRank);
// count how often this process acts as catchment coordinator
CatchmentCoordinatorCount++;
// create the statistics evaluator cache
if (CatchmentStatisticsEvaluatorCache == null)
{
CatchmentStatisticsEvaluatorCache = new Dictionary <string, CatchmentStatisticsEvaluator <ICloneableSimulation, MpiSysConfig> >();
}
}
#endif
}
else
{
Debug.Assert(!workAllocator.RanksByCatchment[catchment.Id].Contains(WorldRank));
}
}
}
public SerialIntracommunicatorProxy(IGroupProxy catchmentGroup)
{
this.catchmentGroup = (SerialGroupProxy)catchmentGroup;
this.wr = this.catchmentGroup.GetWorldRanks();
for (int i = 0; i < wr.Length; i++)
{
groupRank.Add(wr[i], i);
}
}
public MpiIntracommunicatorProxy(IGroupProxy catchmentGroup)
{
this.catchmentGroup = ((MpiGroupProxy)catchmentGroup).group;
communicator = (Intracommunicator)Communicator.world.Create(this.catchmentGroup);
}
internal override IIntracommunicatorProxy CreateIntracommunicatorProxy(IGroupProxy catchmentGroup)
{
return(new SerialIntracommunicatorProxy(catchmentGroup));
}
internal abstract IIntracommunicatorProxy CreateIntracommunicatorProxy(IGroupProxy catchmentGroup);
internal override IIntracommunicatorProxy CreateIntracommunicatorProxy(IGroupProxy catchmentGroup)
{
var res = new MpiIntracommunicatorProxy(catchmentGroup);
return(res.IsNull ? null : res);
}
