/// <summary> /// Entry point for executing a time series model. /// Must only be called on the master process. /// </summary> /// <param name="parameters"> the parameter set for the model run </param> public void Execute(MpiSysConfig parameters) { if (IsSlave) { throw new InvalidOperationException("This method can only be called on the root process"); } Log.DebugFormat("Root: ***** commencing run {0} *****", Iterations); // Tell the slaves to expect some work MpiWorkPacket workPacket = new MpiWorkPacket(SlaveActions.DoWork, parameters); // Gather the catchment results (one result set per catchment) Log.Debug("Root: broadcasting work packets"); WorldBroadcast(ref workPacket, 0); Log.Debug("Root: waiting for catchment results"); catchmentScores = WorldGatherFlattened(new MpiObjectiveScores[0], NumCatchmentResultsPerWorker, 0); Debug.Assert(catchmentScores.Length == GlobalDefinition.Count); Log.DebugFormat("Root: {0} catchment results are in", catchmentScores.Length); // debug info about the ordering of catchments that can be expected in the final results if (Log.IsInfoEnabled && IsFirstRun) { IsFirstRun = false; StringBuilder sb = new StringBuilder(); sb.Append("Catchment orders: "); foreach (var catchmentScore in catchmentScores) { sb.AppendFormat("{0}, ", catchmentScore.CatchmentId); } Log.Info(sb); } Iterations++; }
/// <summary> /// This method must be called by the root process, as RunSlaveSystem sits in an infinite loop. /// Only the root process containing the optimiser knows when the work is complete. /// </summary> public void TerminateSlaves() { // this is required because the Slaves sit in an infinite loop waiting for a new parameter set. Log.Info("Root: Shutting down MPI processes"); MpiWorkPacket workPacket = new MpiWorkPacket(SlaveActions.Terminate); evaluator.WorldBroadcast(ref workPacket, 0); //Communicator.world.Broadcast(ref workPacket, 0); }
internal override void WorldBroadcast(ref MpiWorkPacket workPacket, int root) { // Communicator.world.Broadcast(ref workPacket, root); if (this.rank == 0) { CatchmentResults = new List <MpiObjectiveScores>(); if (workPacket.Command == SlaveActions.DoWork) { // I don't think we can just call DoWork, as happens in the MPI layer (TODO: confirm the intent in the MPI implementation with Daniel). var partialCatchmentResultsByCatchmentIds = new Dictionary <string, SerializableDictionary <string, MpiTimeSeries> > [instances.Count - 1]; var parameters = workPacket.Parameters; for (int i = 1; i < instances.Count; i++) { // execute our list of models, accumulating the results into the appropriate partial result buffer. #if CELL_WEIGHTED_SUMS partialCatchmentResultsByCatchmentIds[i - 1] = EvaluateModels(parameters); #else Dictionary <string, List <SerializableDictionary <string, MpiTimeSeries> > > partialCatchmentResultsByCatchmentId = EvaluateModels(parameters); #endif } for (int i = 1; i < instances.Count; i++) { // For each catchment, accumulate the partial results for each catchment back to the catchment-coordinator. MpiObjectiveScores[] finalCatchmentResults = AccumulateCatchmentResultsInCatchmentCoordinator( partialCatchmentResultsByCatchmentIds[i - 1], parameters); WorldGatherFlattened(finalCatchmentResults, 0); } // OK, all catchment results have been accumulated. Time to send the catchment results back to world root. //Log.DebugFormat("Rank {0}: submitting {1} final catchment results to master", WorldRank, finalCatchmentResults.Length); } } else { throw new NotSupportedException("SerialGriddedCatchmentObjectiveEvaluator only support world broadcast from the master"); } }
/// <summary> /// Entry point for running a slave process. /// </summary> public override void RunSlave() { Log.DebugFormat("Rank {0}: RunSlave", WorldRank); if (IsMaster) { throw new InvalidOperationException("This method can only be called on the slave processes"); } MpiWorkPacket workPacket = new MpiWorkPacket(SlaveActions.Nothing); while (workPacket.Command != SlaveActions.Terminate) { // Check for instructions Log.DebugFormat("Rank {0}: waiting for work", WorldRank); WorldBroadcast(ref workPacket, 0); Log.DebugFormat("Rank {0}: {1}", WorldRank, SlaveActions.ActionNames[workPacket.Command]); if (workPacket.Command == SlaveActions.DoWork) { DoWork(workPacket.Parameters); } } }
internal abstract void WorldBroadcast(ref MpiWorkPacket workPacket, int root);
internal override void WorldBroadcast(ref MpiWorkPacket workPacket, int root) { Communicator.world.Broadcast(ref workPacket, root); }