private async Task InternalFailSubtaskInProgressAsync(int subtaskInProgressId, string[] computationErrors)
{
var failedSubtaskInProgress = await _dbContext.SubtasksInProgress.FindAsync(subtaskInProgressId);
failedSubtaskInProgress.Status = SubtaskInProgressStatus.Error;
failedSubtaskInProgress.Errors = computationErrors;
await _dbContext.SaveChangesAsync();
var failedSubtasksCount = await _dbContext.SubtasksInProgress
.CountAsync(subtaskInProgress =>
subtaskInProgress.SubtaskId == failedSubtaskInProgress.SubtaskId &&
subtaskInProgress.Status == SubtaskInProgressStatus.Error
);
if (failedSubtasksCount >= MaxSubtaskRetries)
{
await FailSubtaskAsync(failedSubtaskInProgress);
await _dbContext.SaveChangesAsync();
}
}
private async Task InternalCompleteSubtaskInProgressAsync(int subtaskInProgressId,
Stream subtaskInProgressResultStream)
{
var subtaskInProgressResult = new byte[subtaskInProgressResultStream.Length];
using (var memoryStream = new MemoryStream(subtaskInProgressResult))
{
await subtaskInProgressResultStream.CopyToAsync(memoryStream);
}
var finishedSubtaskInProgress = await _dbContext.SubtasksInProgress.FindAsync(subtaskInProgressId);
finishedSubtaskInProgress.Status = SubtaskInProgressStatus.Done;
finishedSubtaskInProgress.Result = subtaskInProgressResult;
await _dbContext.SaveChangesAsync();
var isSubtaskFullyComputed = IsSubtaskFullyComputed(finishedSubtaskInProgress.SubtaskId);
if (isSubtaskFullyComputed)
{
await FinishSubtaskAsync(finishedSubtaskInProgress.SubtaskId);
}
}
public async Task <IActionResult> AssignNextAsync([FromBody] AssignNextSubtaskDTO body)
{
if (!Guid.TryParse(body.DistributedNodeId, out var distributedNodeId))
{
return(BadRequest()); // TODO: specify the reason
}
var distributedNode = await _distributedNodeResourceService.GetAsync(distributedNodeId);
if (distributedNode == null)
{
return(NotFound()); // TODO: specify the reason
}
var nextSubtask = await _getNextSubtaskToComputeService.GetNextSubtaskAsync();
if (nextSubtask == null)
{
return(NotFound()); // TODO: specify the reason
}
var subtaskInProgress = new SubtaskInProgress
{
Node = distributedNode,
Status = SubtaskInProgressStatus.Executing,
Subtask = nextSubtask
};
var createdSubtaskInProgress = await _subtaskInProgressResourceService.CreateAsync(subtaskInProgress);
nextSubtask.Status = SubtaskStatus.Executing;
_dbContext.Subtasks.Update(nextSubtask);
await _dbContext.SaveChangesAsync();
var distributedTaskDefinition = await GetSubtasksDistributedTaskDefinition(nextSubtask);
var response = new AssignNextSubtaskResultDTO()
{
CompiledTaskDefinitionURL =
_pathsProvider.GetCompiledTaskDefinitionWebPath(distributedTaskDefinition.DefinitionGuid),
SubtaskId = nextSubtask.StringId,
ProblemPluginInfo = distributedTaskDefinition.ProblemPluginInfo,
SubtaskInProgressId = createdSubtaskInProgress.StringId
};
return(Created($"/subtasks-in-progress/{createdSubtaskInProgress.Id}", response));
}
public async Task CleanAsync()
{
var nodeExpirationDateTime = DateTime.Now.Subtract(_distributedNodeLifetime);
var staleDistributedNodes = await _dbContext.DistributedNodes
.Where(distributedNode => distributedNode.LastKeepAliveTime < nodeExpirationDateTime)
.Include(distributedNode => distributedNode.SubtasksInProgress)
.Select(distributedNode => new
{
DistributedNode = distributedNode,
StaleSubtasksInProgress = distributedNode.SubtasksInProgress.Where(subtaskInProgress =>
subtaskInProgress.Status == SubtaskInProgressStatus.Executing
).ToList(),
ShouldBeRemoved = distributedNode.SubtasksInProgress.All(subtaskInProgress =>
subtaskInProgress.Status == SubtaskInProgressStatus.Cancelled ||
subtaskInProgress.Status == SubtaskInProgressStatus.Error ||
subtaskInProgress.Status == SubtaskInProgressStatus.Executing
)
})
.ToListAsync();
await Task.WhenAll(
staleDistributedNodes.Select(async staleDistributedNode =>
{
await Task.WhenAll(
staleDistributedNode.StaleSubtasksInProgress.Select(
subtaskInProgress =>
_computationCancelService.CancelSubtaskInProgressWithoutSavingAsync(subtaskInProgress.Id)
)
);
if (staleDistributedNode.ShouldBeRemoved)
{
_dbContext.DistributedNodes.Remove(staleDistributedNode.DistributedNode);
}
}));
await _dbContext.SaveChangesAsync();
}
public override async Task <DistributedTask> CreateAsync(DistributedTask resource)
{
var taskDefinition = await GetTaskDefinitionById(resource.DistributedTaskDefinitionId);
await EnsureUniqueName(resource.Name);
DistributedTask distributedTask;
using (var transaction = await _dbContext.Database.BeginTransactionAsync())
{
try
{
distributedTask = await base.CreateAsync(resource);
var subtasks = CreateSubtasks(taskDefinition, distributedTask);
await _dbContext.Subtasks.AddRangeAsync(subtasks);
await _dbContext.SaveChangesAsync();
transaction.Commit();
}
catch (TaskDataParsingException exception)
{
throw new JsonApiException(StatusCodes.Status400BadRequest, exception.Message, exception);
}
catch (TaskDivisionException exception)
{
throw new JsonApiException(StatusCodes.Status400BadRequest, exception.Message, exception);
}
catch (Exception exception)
{
throw new JsonApiException(StatusCodes.Status400BadRequest, "Cannot create the task", exception);
}
}
return(distributedTask);
}
public async Task CancelSubtaskInProgressAsync(int subtaskInProgressId)
{
await CancelSubtaskInProgressWithoutSavingAsync(subtaskInProgressId);
await _dbContext.SaveChangesAsync();
}