Esempio n. 1
0
        /// <summary>
        /// Assigns experiments to available herd agents.
        /// </summary>
        /// <param name="freeHerdAgents"></param>
        public static void AssignExperiments(ref List <ExperimentalUnit> pendingExperiments
                                             , ref List <HerdAgentInfo> freeHerdAgents, ref List <Job> assignedJobs, JobDispatcherOptions options = null)
        {
            //Clear the list: these are jobs which have to be sent
            assignedJobs.Clear();
            //Create a list of agents that are given work. We need to remove them from the "free" list out of the loop
            List <HerdAgentInfo> usedHerdAgents = new List <HerdAgentInfo>();


            //We iterate on the free agents to decide what jobs to give each of them until:
            //  -either there are no more pending experiments
            //  -all agents have been given work
            foreach (HerdAgentInfo agent in freeHerdAgents)
            {
                List <ExperimentalUnit> experiments = new List <ExperimentalUnit>();
                int              numFreeCores       = agent.NumProcessors;
                bool             bAgentUsed         = false;
                ExperimentalUnit experiment;
                bool             bFailedToFindMatch = false;

                //leave one free core??
                if (options != null && options.LeaveOneFreeCore && numFreeCores > 1)
                {
                    numFreeCores--;
                }

                while (numFreeCores > 0 && !bFailedToFindMatch)
                {
                    experiment = FirstFittingExperiment(pendingExperiments, numFreeCores, bAgentUsed, agent);
                    if (experiment != null)
                    {
                        //remove the experiment from the list and add it to running experiments
                        experiments.Add(experiment);
                        pendingExperiments.Remove(experiment);

                        //update the number of free cpu cores
                        if (experiment.RunTimeReqs.NumCPUCores > 0)
                        {
                            numFreeCores -= experiment.RunTimeReqs.NumCPUCores;
                        }
                        else
                        {
                            numFreeCores = 0;
                        }

                        bAgentUsed = true;
                    }
                    else
                    {
                        bFailedToFindMatch = true;
                    }
                }

                if (bAgentUsed)
                {
                    Job newJob = new Job(experiments, agent);
                    assignedJobs.Add(newJob);
                    usedHerdAgents.Add(agent);


                    ++jobId;
                }

                if (pendingExperiments.Count == 0)
                {
                    break;
                }
            }
            //now we can remove used agents from the list
            foreach (HerdAgentInfo agent in usedHerdAgents)
            {
                freeHerdAgents.Remove(agent);
            }
        }
Esempio n. 2
0
        public static async Task <int> RunExperimentsAsync(List <ExperimentalUnit> experiments, List <HerdAgentInfo> freeHerdAgents
                                                           , Monitoring.MsgDispatcher dispatcher
                                                           , CancellationTokenSource cancellationTokenSource
                                                           , JobDispatcherOptions jobDispatcherOptions = null)
        {
            List <Job>         assignedJobs      = new List <Job>();
            List <Task <Job> > monitoredJobTasks = new List <Task <Job> >();
            int numExperimentalUnitsRun          = 0;

            // Calculate run-time requirements
            foreach (ExperimentalUnit experimentalUnit in experiments)
            {
                experimentalUnit.RequestRuntimeRequirements();
            }

            // Assign experiments to free agents
            AssignExperiments(ref experiments, ref freeHerdAgents, ref assignedJobs, jobDispatcherOptions);

            if (assignedJobs.Count == 0)
            {
                return(0);
            }
            try
            {
                while ((assignedJobs.Count > 0 || monitoredJobTasks.Count > 0 ||
                        experiments.Count > 0) && !cancellationTokenSource.IsCancellationRequested)
                {
                    //Create view-models for the jobs and execute them remotely
                    foreach (Job job in assignedJobs)
                    {
                        dispatcher.JobAssigned?.Invoke(job);

                        monitoredJobTasks.Add(job.SendJobAndMonitor(dispatcher));
                    }

                    // All pending experiments sent? Then we await completion to retry in case something fails
                    if (experiments.Count == 0)
                    {
                        Task.WhenAll(monitoredJobTasks).Wait();
                        foreach (Task <Job> task in monitoredJobTasks)
                        {
                            dispatcher.JobFinished?.Invoke(task.Result);
                        }
                        dispatcher.Log?.Invoke("All the experiments have finished");
                        break;
                    }

                    // Wait for the first agent to finish and give it something to do
                    Task <Job> finishedTask = await Task.WhenAny(monitoredJobTasks);

                    Job finishedJob = await finishedTask;
                    dispatcher.Log?.Invoke("Job finished: " + finishedJob.ToString());

                    //A job finished
                    monitoredJobTasks.Remove(finishedTask);
                    if (!cancellationTokenSource.IsCancellationRequested)
                    {
                        numExperimentalUnitsRun += finishedJob.ExperimentalUnits.Count;
                        dispatcher.JobFinished?.Invoke(finishedJob);
                    }


                    if (finishedJob.FailedExperimentalUnits.Count > 0)
                    {
                        experiments.AddRange(finishedJob.FailedExperimentalUnits);
                        dispatcher.Log?.Invoke(finishedJob.FailedExperimentalUnits.Count + " failed experiments enqueued again for further trials");
                    }

                    // Add the herd agent to the free agent list
                    if (!freeHerdAgents.Contains(finishedJob.HerdAgent))
                    {
                        freeHerdAgents.Add(finishedJob.HerdAgent);
                    }

                    // Assign experiments to free agents
                    if (!cancellationTokenSource.IsCancellationRequested)
                    {
                        AssignExperiments(ref experiments, ref freeHerdAgents, ref assignedJobs, jobDispatcherOptions);
                    }
                }
            }
            catch (Exception ex)
            {
                dispatcher.Log?.Invoke("Exception in runExperimentQueueRemotely()");
                dispatcher.Log?.Invoke(ex.StackTrace);
            }
            finally
            {
                if (cancellationTokenSource.IsCancellationRequested)
                {
                    //the user cancelled, need to add unfinished experimental units to the pending list
                    foreach (Job job in assignedJobs)
                    {
                        experiments.AddRange(job.ExperimentalUnits);
                    }
                }
                else
                {
                    foreach (Task <Job> job in monitoredJobTasks)
                    {
                        numExperimentalUnitsRun += job.Result.ExperimentalUnits.Count - job.Result.FailedExperimentalUnits.Count;
                    }
                }
            }
            return(numExperimentalUnitsRun);
        }