public override void AddChildHiveTask(HiveTask hiveTask) { base.AddChildHiveTask(hiveTask); var optimizerHiveTask = (OptimizerHiveTask)hiveTask; syncTasksWithOptimizers = false; if (this.ItemTask != null && optimizerHiveTask.ItemTask != null) { OptimizerTask optimizerTaskClone = (OptimizerTask)optimizerHiveTask.ItemTask.Clone(); if (this.ItemTask.Item is Optimization.Experiment) { if (!this.ItemTask.OptimizerAsExperiment.Optimizers.Contains(optimizerHiveTask.ItemTask.Item)) { UpdateOptimizerInExperiment(this.ItemTask.OptimizerAsExperiment, optimizerHiveTask.ItemTask); } } else if (this.ItemTask.Item is Optimization.BatchRun) { UpdateOptimizerInBatchRun(this.ItemTask.OptimizerAsBatchRun, optimizerHiveTask.ItemTask); } optimizerHiveTask.syncTasksWithOptimizers = false; optimizerHiveTask.ItemTask = optimizerTaskClone; optimizerHiveTask.syncTasksWithOptimizers = true; } syncTasksWithOptimizers = true; }
public override void AddChildHiveTask(HiveTask hiveTask) { base.AddChildHiveTask(hiveTask); var optimizerHiveJob = (OptimizerHiveTask)hiveTask; syncTasksWithOptimizers = false; if (this.ItemTask != null && optimizerHiveJob.ItemTask != null) { // if task is in state Paused, it has to preserve its ResultCollection, which is cleared when a optimizer is added to an experiment OptimizerTask optimizerJobClone = null; if (optimizerHiveJob.Task.State == TaskState.Paused) { optimizerJobClone = (OptimizerTask)optimizerHiveJob.ItemTask.Clone(); } if (this.ItemTask.Item is Optimization.Experiment) { if (!this.ItemTask.OptimizerAsExperiment.Optimizers.Contains(optimizerHiveJob.ItemTask.Item)) { UpdateOptimizerInExperiment(this.ItemTask.OptimizerAsExperiment, optimizerHiveJob.ItemTask); } } else if (this.ItemTask.Item is Optimization.BatchRun) { UpdateOptimizerInBatchRun(this.ItemTask.OptimizerAsBatchRun, optimizerHiveJob.ItemTask); } if (optimizerHiveJob.Task.State == TaskState.Paused) { optimizerHiveJob.ItemTask = optimizerJobClone; } } syncTasksWithOptimizers = true; }
private void jobResultPoller_JobResultReceived(object sender, EventArgs<IEnumerable<LightweightTask>> e) { lock (jobResultReceivedLocker) { foreach (LightweightTask lightweightTask in e.Value) { HiveTask hiveTask = GetHiveTaskById(lightweightTask.Id); if (hiveTask != null) { // lastJobDataUpdate equals DateTime.MinValue right after it was uploaded. When the first results are polled, this value is updated if (hiveTask.Task.State == TaskState.Offline && lightweightTask.State == TaskState.Waiting) { hiveTask.Task.LastTaskDataUpdate = lightweightTask.LastTaskDataUpdate; } hiveTask.UpdateFromLightweightJob(lightweightTask); if (!hiveTask.IsFinishedTaskDownloaded && !hiveTask.IsDownloading && hiveTask.Task.LastTaskDataUpdate < lightweightTask.LastTaskDataUpdate && (lightweightTask.State == TaskState.Finished || lightweightTask.State == TaskState.Aborted || lightweightTask.State == TaskState.Failed || lightweightTask.State == TaskState.Paused)) { log.LogMessage(string.Format("Downloading task {0}", lightweightTask.Id)); hiveTask.IsDownloading = true; jobDownloader.DownloadTaskData(hiveTask.Task, (localJob, itemJob) => { lock (downloadFinishedLocker) { log.LogMessage(string.Format("Finished downloading task {0}", localJob.Id)); HiveTask localHiveTask = GetHiveTaskById(localJob.Id); if (itemJob == null) { // something bad happened to this task. bad task, BAAAD task! localHiveTask.IsDownloading = false; } else { // if the task is paused, download but don't integrate into parent optimizer (to avoid Prepare) if (localJob.State == TaskState.Paused) { localHiveTask.ItemTask = itemJob; } else { if (localJob.ParentTaskId.HasValue) { HiveTask parentHiveTask = GetHiveTaskById(localJob.ParentTaskId.Value); parentHiveTask.IntegrateChild(itemJob, localJob.Id); } else { localHiveTask.ItemTask = itemJob; } } localHiveTask.IsDownloading = false; localHiveTask.Task.LastTaskDataUpdate = lightweightTask.LastTaskDataUpdate; } } }); } } } GC.Collect(); // force GC, because .NET is too lazy here (deserialization takes a lot of memory) if (IsFinished()) { ExecutionState = ExecutionState.Stopped; StopResultPolling(); } else if (IsPaused()) { ExecutionState = ExecutionState.Paused; StopResultPolling(); } else { ExecutionState = ExecutionState.Started; } UpdateTotalExecutionTime(); UpdateStatistics(); OnStateLogListChanged(); OnTaskReceived(); } }
private static void BuildHiveJobTree(HiveTask parentHiveTask, IEnumerable <LightweightTask> allTasks, IDictionary <Guid, HiveTask> allHiveTasks) { IEnumerable <LightweightTask> childTasks = from job in allTasks where job.ParentTaskId.HasValue && job.ParentTaskId.Value == parentHiveTask.Task.Id orderby job.DateCreated ascending select job; foreach (LightweightTask task in childTasks) { HiveTask childHiveTask = allHiveTasks[task.Id]; BuildHiveJobTree(childHiveTask, allTasks, allHiveTasks); parentHiveTask.AddChildHiveTask(childHiveTask); } }
public void StartAsync() { foreach (Guid taskId in taskIds) { taskDownloader.DownloadTaskDataAndTask(taskId, (localTask, itemTask) => { if (localTask != null && itemTask != null) { HiveTask hiveTask = itemTask.CreateHiveTask(); hiveTask.Task = localTask; try { resultsLock.EnterWriteLock(); results.Add(localTask.Id, hiveTask); } finally { resultsLock.ExitWriteLock(); } } }); } }
/// <summary> /// Uploads the given task and all its child-jobs while setting the proper parentJobId values for the childs /// </summary> /// <param name="parentHiveTask">shall be null if its the root task</param> private void UploadTaskWithChildren(IProgress progress, HiveTask hiveTask, HiveTask parentHiveTask, int[] taskCount, int totalJobCount, Guid configPluginId, Guid jobId, ILog log, CancellationToken cancellationToken) { taskUploadSemaphore.WaitOne(); bool semaphoreReleased = false; try { cancellationToken.ThrowIfCancellationRequested(); lock (jobCountLocker) { taskCount[0]++; } TaskData taskData; List <IPluginDescription> plugins; if (hiveTask.ItemTask.ComputeInParallel) { hiveTask.Task.IsParentTask = true; hiveTask.Task.FinishWhenChildJobsFinished = true; taskData = hiveTask.GetAsTaskData(true, out plugins); } else { hiveTask.Task.IsParentTask = false; hiveTask.Task.FinishWhenChildJobsFinished = false; taskData = hiveTask.GetAsTaskData(false, out plugins); } cancellationToken.ThrowIfCancellationRequested(); TryAndRepeat(() => { if (!cancellationToken.IsCancellationRequested) { lock (pluginLocker) { HiveServiceLocator.Instance.CallHiveService((s) => hiveTask.Task.PluginsNeededIds = PluginUtil.GetPluginDependencies(s, this.onlinePlugins, this.alreadyUploadedPlugins, plugins)); } } }, Settings.Default.MaxRepeatServiceCalls, "Failed to upload plugins"); cancellationToken.ThrowIfCancellationRequested(); hiveTask.Task.PluginsNeededIds.Add(configPluginId); hiveTask.Task.JobId = jobId; log.LogMessage(string.Format("Uploading task ({0} kb, {1} objects)", taskData.Data.Count() / 1024, hiveTask.ItemTask.GetObjectGraphObjects().Count())); TryAndRepeat(() => { if (!cancellationToken.IsCancellationRequested) { if (parentHiveTask != null) { hiveTask.Task.Id = HiveServiceLocator.Instance.CallHiveService((s) => s.AddChildTask(parentHiveTask.Task.Id, hiveTask.Task, taskData)); } else { hiveTask.Task.Id = HiveServiceLocator.Instance.CallHiveService((s) => s.AddTask(hiveTask.Task, taskData)); } } }, Settings.Default.MaxRepeatServiceCalls, "Failed to add task", log); cancellationToken.ThrowIfCancellationRequested(); lock (jobCountLocker) { progress.ProgressValue = (double)taskCount[0] / totalJobCount; progress.Message = string.Format("Uploaded task ({0} of {1})", taskCount[0], totalJobCount); } var tasks = new List <TS.Task>(); foreach (HiveTask child in hiveTask.ChildHiveTasks) { var task = TS.Task.Factory.StartNew((tuple) => { var arguments = (Tuple <HiveTask, HiveTask>)tuple; UploadTaskWithChildren(progress, arguments.Item1, arguments.Item2, taskCount, totalJobCount, configPluginId, jobId, log, cancellationToken); }, new Tuple <HiveTask, HiveTask>(child, hiveTask)); task.ContinueWith((x) => log.LogException(x.Exception), TaskContinuationOptions.OnlyOnFaulted); tasks.Add(task); } taskUploadSemaphore.Release(); semaphoreReleased = true; // the semaphore has to be release before waitall! TS.Task.WaitAll(tasks.ToArray()); } finally { if (!semaphoreReleased) { taskUploadSemaphore.Release(); } } }