public override void AddChildHiveTask(HiveTask hiveTask)
{
base.AddChildHiveTask(hiveTask);
var optimizerHiveTask = (OptimizerHiveTask)hiveTask;
syncTasksWithOptimizers = false;
if (this.ItemTask != null && optimizerHiveTask.ItemTask != null)
{
OptimizerTask optimizerTaskClone = (OptimizerTask)optimizerHiveTask.ItemTask.Clone();
if (this.ItemTask.Item is Optimization.Experiment)
{
if (!this.ItemTask.OptimizerAsExperiment.Optimizers.Contains(optimizerHiveTask.ItemTask.Item))
{
UpdateOptimizerInExperiment(this.ItemTask.OptimizerAsExperiment, optimizerHiveTask.ItemTask);
}
}
else if (this.ItemTask.Item is Optimization.BatchRun)
{
UpdateOptimizerInBatchRun(this.ItemTask.OptimizerAsBatchRun, optimizerHiveTask.ItemTask);
}
optimizerHiveTask.syncTasksWithOptimizers = false;
optimizerHiveTask.ItemTask = optimizerTaskClone;
optimizerHiveTask.syncTasksWithOptimizers = true;
}
syncTasksWithOptimizers = true;
}
public override void AddChildHiveTask(HiveTask hiveTask)
{
base.AddChildHiveTask(hiveTask);
var optimizerHiveJob = (OptimizerHiveTask)hiveTask;
syncTasksWithOptimizers = false;
if (this.ItemTask != null && optimizerHiveJob.ItemTask != null)
{
// if task is in state Paused, it has to preserve its ResultCollection, which is cleared when a optimizer is added to an experiment
OptimizerTask optimizerJobClone = null;
if (optimizerHiveJob.Task.State == TaskState.Paused)
{
optimizerJobClone = (OptimizerTask)optimizerHiveJob.ItemTask.Clone();
}
if (this.ItemTask.Item is Optimization.Experiment)
{
if (!this.ItemTask.OptimizerAsExperiment.Optimizers.Contains(optimizerHiveJob.ItemTask.Item))
{
UpdateOptimizerInExperiment(this.ItemTask.OptimizerAsExperiment, optimizerHiveJob.ItemTask);
}
}
else if (this.ItemTask.Item is Optimization.BatchRun)
{
UpdateOptimizerInBatchRun(this.ItemTask.OptimizerAsBatchRun, optimizerHiveJob.ItemTask);
}
if (optimizerHiveJob.Task.State == TaskState.Paused)
{
optimizerHiveJob.ItemTask = optimizerJobClone;
}
}
syncTasksWithOptimizers = true;
}
private void jobResultPoller_JobResultReceived(object sender, EventArgs<IEnumerable<LightweightTask>> e) {
lock (jobResultReceivedLocker) {
foreach (LightweightTask lightweightTask in e.Value) {
HiveTask hiveTask = GetHiveTaskById(lightweightTask.Id);
if (hiveTask != null) {
// lastJobDataUpdate equals DateTime.MinValue right after it was uploaded. When the first results are polled, this value is updated
if (hiveTask.Task.State == TaskState.Offline && lightweightTask.State == TaskState.Waiting) {
hiveTask.Task.LastTaskDataUpdate = lightweightTask.LastTaskDataUpdate;
}
hiveTask.UpdateFromLightweightJob(lightweightTask);
if (!hiveTask.IsFinishedTaskDownloaded && !hiveTask.IsDownloading && hiveTask.Task.LastTaskDataUpdate < lightweightTask.LastTaskDataUpdate && (lightweightTask.State == TaskState.Finished || lightweightTask.State == TaskState.Aborted || lightweightTask.State == TaskState.Failed || lightweightTask.State == TaskState.Paused)) {
log.LogMessage(string.Format("Downloading task {0}", lightweightTask.Id));
hiveTask.IsDownloading = true;
jobDownloader.DownloadTaskData(hiveTask.Task, (localJob, itemJob) => {
lock (downloadFinishedLocker) {
log.LogMessage(string.Format("Finished downloading task {0}", localJob.Id));
HiveTask localHiveTask = GetHiveTaskById(localJob.Id);
if (itemJob == null) {
// something bad happened to this task. bad task, BAAAD task!
localHiveTask.IsDownloading = false;
} else {
// if the task is paused, download but don't integrate into parent optimizer (to avoid Prepare)
if (localJob.State == TaskState.Paused) {
localHiveTask.ItemTask = itemJob;
} else {
if (localJob.ParentTaskId.HasValue) {
HiveTask parentHiveTask = GetHiveTaskById(localJob.ParentTaskId.Value);
parentHiveTask.IntegrateChild(itemJob, localJob.Id);
} else {
localHiveTask.ItemTask = itemJob;
}
}
localHiveTask.IsDownloading = false;
localHiveTask.Task.LastTaskDataUpdate = lightweightTask.LastTaskDataUpdate;
}
}
});
}
}
}
GC.Collect(); // force GC, because .NET is too lazy here (deserialization takes a lot of memory)
if (IsFinished()) {
ExecutionState = ExecutionState.Stopped;
StopResultPolling();
} else if (IsPaused()) {
ExecutionState = ExecutionState.Paused;
StopResultPolling();
} else {
ExecutionState = ExecutionState.Started;
}
UpdateTotalExecutionTime();
UpdateStatistics();
OnStateLogListChanged();
OnTaskReceived();
}
}
private static void BuildHiveJobTree(HiveTask parentHiveTask, IEnumerable <LightweightTask> allTasks, IDictionary <Guid, HiveTask> allHiveTasks)
{
IEnumerable <LightweightTask> childTasks = from job in allTasks
where job.ParentTaskId.HasValue && job.ParentTaskId.Value == parentHiveTask.Task.Id
orderby job.DateCreated ascending
select job;
foreach (LightweightTask task in childTasks)
{
HiveTask childHiveTask = allHiveTasks[task.Id];
BuildHiveJobTree(childHiveTask, allTasks, allHiveTasks);
parentHiveTask.AddChildHiveTask(childHiveTask);
}
}
public void StartAsync() {
foreach (Guid taskId in taskIds) {
taskDownloader.DownloadTaskDataAndTask(taskId,
(localTask, itemTask) => {
if (localTask != null && itemTask != null) {
HiveTask hiveTask = itemTask.CreateHiveTask();
hiveTask.Task = localTask;
try {
resultsLock.EnterWriteLock();
results.Add(localTask.Id, hiveTask);
}
finally { resultsLock.ExitWriteLock(); }
}
});
}
}
/// <summary>
/// Uploads the given task and all its child-jobs while setting the proper parentJobId values for the childs
/// </summary>
/// <param name="parentHiveTask">shall be null if its the root task</param>
private void UploadTaskWithChildren(IProgress progress, HiveTask hiveTask, HiveTask parentHiveTask, int[] taskCount, int totalJobCount, Guid configPluginId, Guid jobId, ILog log, CancellationToken cancellationToken)
{
taskUploadSemaphore.WaitOne();
bool semaphoreReleased = false;
try {
cancellationToken.ThrowIfCancellationRequested();
lock (jobCountLocker) {
taskCount[0]++;
}
TaskData taskData;
List <IPluginDescription> plugins;
if (hiveTask.ItemTask.ComputeInParallel)
{
hiveTask.Task.IsParentTask = true;
hiveTask.Task.FinishWhenChildJobsFinished = true;
taskData = hiveTask.GetAsTaskData(true, out plugins);
}
else
{
hiveTask.Task.IsParentTask = false;
hiveTask.Task.FinishWhenChildJobsFinished = false;
taskData = hiveTask.GetAsTaskData(false, out plugins);
}
cancellationToken.ThrowIfCancellationRequested();
TryAndRepeat(() => {
if (!cancellationToken.IsCancellationRequested)
{
lock (pluginLocker) {
HiveServiceLocator.Instance.CallHiveService((s) => hiveTask.Task.PluginsNeededIds = PluginUtil.GetPluginDependencies(s, this.onlinePlugins, this.alreadyUploadedPlugins, plugins));
}
}
}, Settings.Default.MaxRepeatServiceCalls, "Failed to upload plugins");
cancellationToken.ThrowIfCancellationRequested();
hiveTask.Task.PluginsNeededIds.Add(configPluginId);
hiveTask.Task.JobId = jobId;
log.LogMessage(string.Format("Uploading task ({0} kb, {1} objects)", taskData.Data.Count() / 1024, hiveTask.ItemTask.GetObjectGraphObjects().Count()));
TryAndRepeat(() => {
if (!cancellationToken.IsCancellationRequested)
{
if (parentHiveTask != null)
{
hiveTask.Task.Id = HiveServiceLocator.Instance.CallHiveService((s) => s.AddChildTask(parentHiveTask.Task.Id, hiveTask.Task, taskData));
}
else
{
hiveTask.Task.Id = HiveServiceLocator.Instance.CallHiveService((s) => s.AddTask(hiveTask.Task, taskData));
}
}
}, Settings.Default.MaxRepeatServiceCalls, "Failed to add task", log);
cancellationToken.ThrowIfCancellationRequested();
lock (jobCountLocker) {
progress.ProgressValue = (double)taskCount[0] / totalJobCount;
progress.Message = string.Format("Uploaded task ({0} of {1})", taskCount[0], totalJobCount);
}
var tasks = new List <TS.Task>();
foreach (HiveTask child in hiveTask.ChildHiveTasks)
{
var task = TS.Task.Factory.StartNew((tuple) => {
var arguments = (Tuple <HiveTask, HiveTask>)tuple;
UploadTaskWithChildren(progress, arguments.Item1, arguments.Item2, taskCount, totalJobCount, configPluginId, jobId, log, cancellationToken);
}, new Tuple <HiveTask, HiveTask>(child, hiveTask));
task.ContinueWith((x) => log.LogException(x.Exception), TaskContinuationOptions.OnlyOnFaulted);
tasks.Add(task);
}
taskUploadSemaphore.Release(); semaphoreReleased = true; // the semaphore has to be release before waitall!
TS.Task.WaitAll(tasks.ToArray());
} finally {
if (!semaphoreReleased)
{
taskUploadSemaphore.Release();
}
}
}
