/// <summary>
/// Adds a new Azure Batch pool/job/task for the given <see cref="TesTask"/>
/// </summary>
/// <param name="tesTask">The <see cref="TesTask"/> to schedule on Azure Batch</param>
/// <returns>A task to await</returns>
private async Task AddBatchJobAsync(TesTask tesTask)
{
try
{
var jobId = await azureProxy.GetNextBatchJobIdAsync(tesTask.Id);
var virtualMachineInfo = await GetVmSizeAsync(tesTask.Resources);
await CheckBatchAccountQuotas((int)tesTask.Resources.CpuCores.GetValueOrDefault(DefaultCoreCount), virtualMachineInfo.LowPriority);
// TODO?: Support for multiple executors. Cromwell has single executor per task.
var dockerImage = tesTask.Executors.First().Image;
var cloudTask = await ConvertTesTaskToBatchTaskAsync(tesTask);
var poolInformation = await CreatePoolInformation(dockerImage, virtualMachineInfo.VmSize, virtualMachineInfo.LowPriority);
tesTask.Resources.VmInfo = virtualMachineInfo;
logger.LogInformation($"Creating batch job for TES task {tesTask.Id}. Using VM size {virtualMachineInfo}.");
await azureProxy.CreateBatchJobAsync(jobId, cloudTask, poolInformation);
tesTask.State = TesState.INITIALIZINGEnum;
}
catch (AzureBatchQuotaMaxedOutException exception)
{
logger.LogInformation($"Not enough quota available for task Id {tesTask.Id}. Reason: {exception.Message}. Task will remain in queue.");
}
catch (Exception exc)
{
tesTask.State = TesState.SYSTEMERROREnum;
tesTask.WriteToSystemLog(exc.Message, exc.StackTrace);
logger.LogError(exc, exc.Message);
}
}
/// <summary>
/// Transitions the <see cref="TesTask"/> to the new state, based on the rules defined in the tesTaskStateTransitions list.
/// </summary>
/// <param name="tesTask">TES task</param>
/// <param name="batchTaskState">Current Azure Batch task state</param>
/// <param name="executionInfo">Batch job execution info</param>
/// <returns>True if the TES task was changed.</returns>
private async Task <bool> HandleTesTaskTransitionAsync(TesTask tesTask, BatchTaskState batchTaskState, string executionInfo)
{
var tesTaskChanged = false;
var mapItem = tesTaskStateTransitions
.FirstOrDefault(m => (m.Condition == null || m.Condition(tesTask)) && (m.CurrentBatchTaskState == null || m.CurrentBatchTaskState == batchTaskState));
if (mapItem != null)
{
if (mapItem.Action != null)
{
await mapItem.Action(tesTask);
if (executionInfo != null)
{
tesTask.WriteToSystemLog(executionInfo);
}
tesTaskChanged = true;
}
if (mapItem.NewTesTaskState != null && mapItem.NewTesTaskState != tesTask.State)
{
tesTask.State = mapItem.NewTesTaskState.Value;
if (executionInfo != null)
{
tesTask.WriteToSystemLog(executionInfo);
}
tesTaskChanged = true;
}
}
return(tesTaskChanged);
}