public T.Task <List <InternalTask> > GenerateTasksAsync(Job job, CancellationToken token) { return(T.Task.FromResult(Enumerable.Range(1, job.TargetNodes.Length).Select(id => { var t = InternalTask.CreateFrom(job); t.CustomizedData = t.Node = job.TargetNodes[id - 1]; t.Id = id; t.MaximumRuntimeSeconds = job.DefaultTaskMaximumRuntimeSeconds; return t; }).ToList())); }
public T.Task <DispatchResult> DispatchAsync(Job job, CancellationToken token) { return(T.Task.FromResult(new DispatchResult() { ModifiedJob = job, Tasks = Enumerable.Range(1, job.TargetNodes.Length).Select(id => { var t = InternalTask.CreateFrom(job); t.CustomizedData = t.Node = job.TargetNodes[id - 1]; t.Id = id; t.MaximumRuntimeSeconds = job.MaximumRuntimeSeconds; return t; }).ToList() })); }
public override async T.Task ProcessAsync(Job job, JobEventMessage message, CancellationToken token) { var jobTable = this.Utilities.GetJobsTable(); if (job.State != JobState.Queued) { this.Logger.Error("The job {0} state {1} is not queued.", job.Id, job.State); return; } var tasks = await this.JobTypeHandler.GenerateTasksAsync(job, token); if (tasks == null) { this.Logger.Error("The job {0} script doesn't generate any tasks", job.Id); await this.Utilities.UpdateJobAsync(job.Type, job.Id, j => { (j.Events ?? (j.Events = new List <Event>())).Add(new Event() { Content = $"The job {job.Id} script doesn't generate any tasks.", Source = EventSource.Job, Type = EventType.Alert, }); j.State = JobState.Failed; j.TaskCount = 0; }, token, this.Logger); return; } var allParentIds = new HashSet <int>(tasks.SelectMany(t => t.ParentIds ?? new List <int>())); var endingIds = tasks.Where(t => !allParentIds.Contains(t.Id)).Select(t => t.Id).ToList(); var startTask = InternalTask.CreateFrom(job); startTask.Id = 0; startTask.CustomizedData = InternalTask.StartTaskMark; tasks.ForEach(t => { if (t.ParentIds == null || t.ParentIds.Count == 0) { t.ParentIds = new List <int>() { startTask.Id } } ; t.ChildIds?.Clear(); }); var endTask = InternalTask.CreateFrom(job); endTask.Id = int.MaxValue; endTask.CustomizedData = InternalTask.EndTaskMark; endTask.ParentIds = endingIds; this.Logger.Information("Job {0} task {1} has {2} parent ids, {3}", job.Id, endTask.Id, endTask.ParentIds.Count, string.Join(",", endTask.ParentIds)); tasks.Add(startTask); tasks.Add(endTask); var(success, msg) = this.FillData(tasks, job); if (!success) { this.Logger.Error(msg); await this.Utilities.UpdateJobAsync(job.Type, job.Id, j => { j.State = JobState.Failed; (j.Events ?? (j.Events = new List <Event>())).Add(new Event() { Content = msg, Source = EventSource.Job, Type = EventType.Alert }); }, token, this.Logger); return; } const int MaxChildIds = 1000; this.Logger.Information("Job {0} Converting {1} Tasks to Instances.", job.Id, tasks.Count); var taskInstances = tasks.Select(it => { string zipppedParentIds = Compress.GZip(string.Join(",", it.ParentIds ?? new List <int>())); var childIds = it.ChildIds; childIds = childIds ?? new List <int>(); childIds = childIds.Count > MaxChildIds ? null : childIds; return(new Task() { ChildIds = childIds, ZippedParentIds = zipppedParentIds, CommandLine = it.CommandLine, CustomizedData = it.CustomizedData, Id = it.Id, JobId = it.JobId, JobType = it.JobType, Node = it.Node, RequeueCount = it.RequeueCount, State = string.Equals(it.CustomizedData, Task.StartTaskMark, StringComparison.OrdinalIgnoreCase) ? TaskState.Finished : TaskState.Queued, MaximumRuntimeSeconds = it.MaximumRuntimeSeconds, }); }).ToList(); var childIdsContent = tasks .Where(it => (it.ChildIds?.Count ?? 0) > MaxChildIds) .Select(it => new { it.Id, it.JobId, it.RequeueCount, it.ChildIds, }) .ToList(); this.Logger.Information("Job {0} Converting {1} Tasks to TaskStartInfo.", job.Id, tasks.Count); var taskInfos = tasks.Select(it => new TaskStartInfo() { Id = it.Id, JobId = it.JobId, JobType = it.JobType, NodeName = it.Node, Password = it.Password, PrivateKey = it.PrivateKey, PublicKey = it.PublicKey, UserName = it.UserName, StartInfo = new ProcessStartInfo(it.CommandLine, it.WorkingDirectory, null, null, null, it.EnvironmentVariables, null, it.RequeueCount), }).ToList(); this.Logger.Information("Job {0} Inserting {1} Tasks to Table.", job.Id, tasks.Count); var jobPartitionKey = this.Utilities.GetJobPartitionKey(job.Type, job.Id); await jobTable.InsertOrReplaceBatchAsync(token, taskInstances.Select(t => new JsonTableEntity( jobPartitionKey, this.Utilities.GetTaskKey(job.Id, t.Id, job.RequeueCount), t)).ToArray()); if (childIdsContent.Select(cid => cid.Id).Distinct().Count() != childIdsContent.Count) { await this.Utilities.UpdateJobAsync(job.Type, job.Id, j => { j.State = JobState.Failed; (j.Events ?? (j.Events = new List <Event>())).Add(new Event() { Content = $"Duplicate task ids found.", Source = EventSource.Job, Type = EventType.Alert, }); }, token, this.Logger); return; } this.Logger.Information("Job {0} Uploading {1} Tasks child ids content to blob.", job.Id, childIdsContent.Count); await T.Task.WhenAll(childIdsContent.Select(async childIds => { var taskKey = this.Utilities.GetTaskKey(childIds.JobId, childIds.Id, childIds.RequeueCount); var childIdsBlob = await this.Utilities.CreateOrReplaceTaskChildrenBlobAsync(taskKey, token); var jsonContent = JsonConvert.SerializeObject(childIds.ChildIds); await childIdsBlob.UploadTextAsync(jsonContent, Encoding.UTF8, null, null, null, token); })); this.Logger.Information("Job {0} Inserting {1} TaskInfo to Table.", job.Id, taskInfos.Count); await jobTable.InsertOrReplaceBatchAsync(token, taskInfos.Select(t => new JsonTableEntity( jobPartitionKey, this.Utilities.GetTaskInfoKey(job.Id, t.Id, job.RequeueCount), t)).ToArray()); this.Logger.Information("Job {0} updating job status.", job.Id); JobState state = JobState.Queued; await this.Utilities.UpdateJobAsync(job.Type, job.Id, j => { state = j.State = (j.State == JobState.Queued ? JobState.Running : j.State); j.TaskCount = taskInstances.Count - 2; }, token, this.Logger); if (state == JobState.Running) { this.Logger.Information("Job {0} Starting the job", job.Id); async T.Task addFirstTask() { var taskCompletionQueue = await this.Utilities.GetOrCreateJobTaskCompletionQueueAsync(job.Id, token); await taskCompletionQueue.AddMessageAsync(new CloudQueueMessage( JsonConvert.SerializeObject(new TaskCompletionMessage() { JobId = job.Id, Id = 0, JobType = job.Type, RequeueCount = job.RequeueCount, ChildIds = startTask.ChildIds })), null, null, null, null, token); }; async T.Task addRunningJob() { var runningJobQueue = this.Utilities.GetRunningJobQueue(); await runningJobQueue.AddMessageAsync(new CloudQueueMessage( JsonConvert.SerializeObject(new RunningJobMessage() { JobId = job.Id, JobType = job.Type, RequeueCount = job.RequeueCount })), null, null, null, null, token); }; await T.Task.WhenAll(addFirstTask(), addRunningJob()); } }