/// <summary>
/// Retrieves all actionable TES tasks from the database, performs an action in the batch system, and updates the resultant state
/// </summary>
/// <returns></returns>
private async Task OrchestrateTesTasksOnBatch()
{
string continuationToken = null;
do
{
IEnumerable <RepositoryItem <TesTask> > tesTasks;
(continuationToken, tesTasks) = await repository.GetItemsAsync(
predicate : t => t.State == TesState.QUEUEDEnum ||
t.State == TesState.INITIALIZINGEnum ||
t.State == TesState.RUNNINGEnum ||
(t.State == TesState.CANCELEDEnum && t.IsCancelRequested),
pageSize : 256,
continuationToken : continuationToken);
foreach (var tesTask in tesTasks)
{
try
{
var isModified = await batchScheduler.ProcessTesTaskAsync(tesTask.Value);
if (isModified)
{
//task has transitioned
if (tesTask.Value.State == TesState.CANCELEDEnum ||
tesTask.Value.State == TesState.COMPLETEEnum ||
tesTask.Value.State == TesState.EXECUTORERROREnum ||
tesTask.Value.State == TesState.SYSTEMERROREnum)
{
tesTask.Value.EndTime = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffzzz", DateTimeFormatInfo.InvariantInfo);
if (tesTask.Value.State == TesState.EXECUTORERROREnum || tesTask.Value.State == TesState.SYSTEMERROREnum)
{
logger.LogDebug($"{tesTask.Value.Id} is in a terminal state: {tesTask.Value.State}");
}
}
await repository.UpdateItemAsync(tesTask.Value.Id, new RepositoryItem <TesTask> {
ETag = tesTask.ETag, Value = tesTask.Value
});
}
}
catch (Exception exc)
{
if (++tesTask.Value.ErrorCount > 3) // TODO: Should we increment this for exceptions here (current behaviour) or the attempted executions on the batch?
{
tesTask.Value.State = TesState.SYSTEMERROREnum;
tesTask.Value.EndTime = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffzzz", DateTimeFormatInfo.InvariantInfo);
tesTask.Value.WriteToSystemLog(exc.Message, exc.StackTrace);
}
logger.LogError(exc, $"TES Task '{tesTask.Value.Id}' threw an exception.");
await repository.UpdateItemAsync(tesTask.Value.Id, tesTask);
}
}
}while (continuationToken != null);
}
/// <summary>
/// Retrieves all actionable TES tasks from the database, performs an action in the batch system, and updates the resultant state
/// </summary>
/// <returns></returns>
private async Task OrchestrateTesTasksOnBatch(CancellationToken cancellationToken) // TODO: implement
{
var tesTasks = (await repository.GetItemsAsync(
predicate: t => t.State == TesState.QUEUEDEnum ||
t.State == TesState.INITIALIZINGEnum ||
t.State == TesState.RUNNINGEnum ||
(t.State == TesState.CANCELEDEnum && t.IsCancelRequested)))
.ToList();
if (!tesTasks.Any())
{
return;
}
var startTime = DateTime.UtcNow;
foreach (var tesTask in tesTasks)
{
try
{
var isModified = await batchScheduler.ProcessTesTaskAsync(tesTask);
if (isModified)
{
//task has transitioned
if (tesTask.State == TesState.CANCELEDEnum ||
tesTask.State == TesState.COMPLETEEnum ||
tesTask.State == TesState.EXECUTORERROREnum ||
tesTask.State == TesState.SYSTEMERROREnum)
{
tesTask.EndTime = DateTimeOffset.UtcNow;
if (tesTask.State == TesState.EXECUTORERROREnum || tesTask.State == TesState.SYSTEMERROREnum)
{
logger.LogDebug($"{tesTask.Id} failed, state: {tesTask.State}, reason: {tesTask.FailureReason}");
}
}
await repository.UpdateItemAsync(tesTask);
}
}
catch (Exception exc)
{
if (++tesTask.ErrorCount > 3) // TODO: Should we increment this for exceptions here (current behaviour) or the attempted executions on the batch?
{
tesTask.State = TesState.SYSTEMERROREnum;
tesTask.EndTime = DateTimeOffset.UtcNow;
tesTask.SetFailureReason("UnknownError", exc.Message, exc.StackTrace);
}
logger.LogError(exc, $"TES Task '{tesTask.Id}' threw an exception.");
await repository.UpdateItemAsync(tesTask);
}
}
logger.LogDebug($"OrchestrateTesTasksOnBatch for {tesTasks.Count} tasks completed in {DateTime.UtcNow.Subtract(startTime).TotalSeconds} seconds.");
}
