public async T.Task <DispatchResult> DispatchAsync(Job job, CancellationToken token) { // TODO: github integration var jobTable = this.Utilities.GetJobsTable(); this.Logger.Information("Generating tasks for job {0}", job.Id); var diagTest = await jobTable.RetrieveAsync <InternalDiagnosticsTest>(this.Utilities.GetDiagPartitionKey(job.DiagnosticTest.Category), job.DiagnosticTest.Name, token); if (diagTest != null) { var scriptBlob = this.Utilities.GetBlob(diagTest.DispatchScript.ContainerName, diagTest.DispatchScript.Name); var nodesTable = this.Utilities.GetNodesTable(); var metadataKey = this.Utilities.GetMetadataKey(); this.Logger.Information("GenerateTasks, Querying node info for job {0}", job.Id); var metadata = await T.Task.WhenAll(job.TargetNodes.Select(async n => { var heartbeatKey = this.Utilities.GetHeartbeatKey(n); var nodeInfo = await nodesTable.RetrieveJsonTableEntityAsync(this.Utilities.NodesPartitionKey, heartbeatKey, token); return(new { Node = n, Metadata = await nodesTable.RetrieveAsync <object>(this.Utilities.GetNodePartitionKey(n), metadataKey, token), Heartbeat = nodeInfo?.GetObject <ComputeClusterNodeInformation>(), LastHeartbeatTime = nodeInfo?.Timestamp, NodeRegistrationInfo = await nodesTable.RetrieveAsync <ComputeClusterRegistrationInformation>(this.Utilities.NodesPartitionKey, this.Utilities.GetRegistrationKey(n), token), }); })); var dispatchTasks = await PythonExecutor.ExecuteAsync(scriptBlob, new { Job = job, Nodes = metadata }, token); if (dispatchTasks.IsError) { this.Logger.Error("Generate tasks, job {0}, {1}", job.Id, dispatchTasks.ErrorMessage); await this.Utilities.FailJobWithEventAsync( job, $"Dispatch script {dispatchTasks.ErrorMessage}", token); return(null); } else { return(JsonConvert.DeserializeObject <DispatchResult>(dispatchTasks.Output)); } } else { await this.Utilities.FailJobWithEventAsync( job, $"No diag test {job.DiagnosticTest.Category}/{job.DiagnosticTest.Name} found", token); this.Logger.Error("No diag test found"); return(null); } }
public async T.Task <string> AggregateTasksAsync(Job job, List <Task> tasks, List <ComputeClusterTaskInformation> taskResults, CancellationToken token) { var jobTable = this.Utilities.GetJobsTable(); this.Logger.Information("AggregateTasks, job {0}", job.Id); var diagTest = await jobTable.RetrieveAsync <InternalDiagnosticsTest>( this.Utilities.GetDiagPartitionKey(job.DiagnosticTest.Category), job.DiagnosticTest.Name, token); string result = string.Empty; if (diagTest != null) { var scriptBlob = this.Utilities.GetBlob(diagTest.AggregationScript.ContainerName, diagTest.AggregationScript.Name); var aggregationResult = await PythonExecutor.ExecuteAsync(scriptBlob, new { Job = job, Tasks = tasks, TaskResults = taskResults }, token); result = aggregationResult.Output; if (aggregationResult.IsError) { this.Logger.Error("AggregateTasks, job {0}, {1}", job.Id, aggregationResult.ErrorMessage); (job.Events ?? (job.Events = new List <Event>())).Add(new Event() { Content = $"Diag reduce script {aggregationResult.ErrorMessage}", Source = EventSource.Job, Type = EventType.Alert, }); job.State = JobState.Failed; } } else { job.State = JobState.Failed; (job.Events ?? (job.Events = new List <Event>())).Add(new Event() { Content = $"No diag test {job.DiagnosticTest.Category}/{job.DiagnosticTest.Name} found", Source = EventSource.Job, Type = EventType.Alert }); } return(result); }
private async T.Task <bool> TaskResultHook( Job job, int taskId, string path, CancellationToken token) { var jobPartitionKey = this.Utilities.GetJobPartitionKey(job.Type, job.Id); var taskResultKey = this.Utilities.GetTaskResultKey(job.Id, taskId, job.RequeueCount); var taskInfo = await this.jobsTable.RetrieveAsync <ComputeClusterTaskInformation>(jobPartitionKey, taskResultKey, token); if (taskInfo == null) { this.Logger.Information("There is no task info recorded for job {0}, task {1}, requeue {2}, skip the filter.", job.Id, taskId, job.RequeueCount); return(true); } var filteredResult = await PythonExecutor.ExecuteAsync(path, new { Job = job, Task = taskInfo }, token); this.Logger.Information("Task filter script execution for job {0}, task {1}, filteredResult exit code {2}, result length {3}", job.Id, taskId, filteredResult.ExitCode, filteredResult.Output?.Length); taskInfo.Message = filteredResult.IsError ? taskInfo.Message : filteredResult.Output; await this.jobsTable.InsertOrReplaceAsync(jobPartitionKey, taskResultKey, taskInfo, token); if (filteredResult.IsError) { this.Logger.Error("There is an error in task filter script. {0}", filteredResult.ErrorMessage); await this.Utilities.UpdateJobAsync(job.Type, job.Id, j => { (j.Events ?? (j.Events = new List <Event>())).Add(new Event() { Content = filteredResult.ErrorMessage, Source = EventSource.Job, Type = EventType.Alert, }); j.State = JobState.Failed; }, token, this.Logger); return(true); } return(true); }
public async T.Task <List <InternalTask> > GenerateTasksAsync(Job job, CancellationToken token) { // TODO: github integration var jobTable = this.Utilities.GetJobsTable(); this.Logger.Information("Generating tasks for job {0}", job.Id); var diagTest = await jobTable.RetrieveAsync <InternalDiagnosticsTest>(this.Utilities.GetDiagPartitionKey(job.DiagnosticTest.Category), job.DiagnosticTest.Name, token); if (diagTest != null) { var scriptBlob = this.Utilities.GetBlob(diagTest.DispatchScript.ContainerName, diagTest.DispatchScript.Name); var nodesTable = this.Utilities.GetNodesTable(); var metadataKey = this.Utilities.GetMetadataKey(); this.Logger.Information("GenerateTasks, Querying node info for job {0}", job.Id); var metadata = await T.Task.WhenAll(job.TargetNodes.Select(async n => new { Node = n, Metadata = await nodesTable.RetrieveAsync <object>(this.Utilities.GetNodePartitionKey(n), metadataKey, token), NodeRegistrationInfo = await nodesTable.RetrieveAsync <ComputeClusterRegistrationInformation>(this.Utilities.NodesPartitionKey, this.Utilities.GetRegistrationKey(n), token), })); var dispatchTasks = await PythonExecutor.ExecuteAsync(scriptBlob, new { Job = job, Nodes = metadata }, token); if (dispatchTasks.IsError) { this.Logger.Error("Generate tasks, job {0}, {1}", job.Id, dispatchTasks.ErrorMessage); await this.Utilities.UpdateJobAsync(job.Type, job.Id, j => { j.State = JobState.Failed; (j.Events ?? (j.Events = new List <Event>())).Add(new Event() { Content = $"Dispatch script {dispatchTasks.ErrorMessage}", Source = EventSource.Job, Type = EventType.Alert }); }, token, this.Logger); return(null); } else { return(JsonConvert.DeserializeObject <List <InternalTask> >(dispatchTasks.Output)); } } else { await this.Utilities.UpdateJobAsync(job.Type, job.Id, j => { j.State = JobState.Failed; (j.Events ?? (j.Events = new List <Event>())).Add(new Event() { Content = $"No diag test {job.DiagnosticTest.Category}/{job.DiagnosticTest.Name} found", Source = EventSource.Job, Type = EventType.Alert }); }, token, this.Logger); this.Logger.Error("No diag test found"); return(null); } }