/// <summary>
        /// Creates the catchment communicators.
        /// </summary>
        /// <param name="workAllocator">The work allocator.</param>
        private void CreateCommunicators(IWorkAllocator workAllocator)
        {
            // create the communicators (creating communicators is a collective operation, all processes that belong to the new communicator
            // must participate in the call).
            // It is also important that all processes create the communicators in the same order to avoid deadlock
            // There is one communicator per catchment
            Log.DebugFormat("Rank {0}: Creating communicators", WorldRank);

            // we need to sort the list of catchments by catchment CatchmentId.
            // All processes must have the same catchment order or they can deadlock when creating communicators that span processes.
            GlobalDefinition.SortCatchmentsById();

            foreach (CatchmentDefinition catchment in GlobalDefinition)
            {
                Log.DebugFormat("Rank {0}: catchment {1} Creating communicator for {2} processes", WorldRank, catchment.Id, workAllocator.RanksByCatchment[catchment.Id].Count);
                IGroupProxy catchmentGroup = CreateGroup(workAllocator.RanksByCatchment[catchment.Id].ToArray());
                Log.DebugFormat("Rank {0}: Catchment group created, size = {1}", WorldRank, catchmentGroup.Size);
                IIntracommunicatorProxy catchmentCommunicator = CreateIntracommunicatorProxy(catchmentGroup);
                if (catchmentCommunicator != null)
                {
                    Log.DebugFormat("Rank {0}: Communicator created, rank = {1} size = {2}", WorldRank, catchmentCommunicator.GetRank(this.WorldRank), catchmentCommunicator.Size);
                }
                else
                {
                    Log.DebugFormat("Rank {0}: Communicator created, I am not a member", WorldRank);
                }

                // catchmentCommunicator will be null if the current rank is not a member of the catchmentGroup.
                // This is OK, as each rank only requires the communicators for catchments it is involved in.
                if (catchmentCommunicator != null)
                {
                    Debug.Assert(workAllocator.RanksByCatchment[catchment.Id].Contains(WorldRank));
                    communicatorsByCatchmentId.Add(catchment.Id, catchmentCommunicator);

#if CELL_WEIGHTED_SUMS
                    // If I am the catchment coordinator for at least one catchment, then I will need the dictionary of cached
                    // catchment statistics evaluators
                    if (catchmentCommunicator.GetRank(this.WorldRank) == 0)
                    {
                        Log.DebugFormat("Rank {0}: I am catchment coordinator. Creating stats evaluator cache.", WorldRank);

                        // count how often this process acts as catchment coordinator
                        CatchmentCoordinatorCount++;

                        // create the statistics evaluator cache
                        if (CatchmentStatisticsEvaluatorCache == null)
                        {
                            CatchmentStatisticsEvaluatorCache = new Dictionary <string, CatchmentStatisticsEvaluator <ICloneableSimulation, MpiSysConfig> >();
                        }
                    }
#endif
                }
                else
                {
                    Debug.Assert(!workAllocator.RanksByCatchment[catchment.Id].Contains(WorldRank));
                }
            }
        }