/// <summary>
/// Default constructor
/// </summary>
/// <param name="configuration">The configuration instance settings</param>
/// <param name="azureProxy">Azure Proxy</param>
/// <param name="repository">The main TES task database repository</param>
/// <param name="logger">The logger instance</param>
public DeleteCompletedBatchJobsHostedService(IConfiguration configuration, IAzureProxy azureProxy, IRepository <TesTask> repository, ILogger <DeleteCompletedBatchJobsHostedService> logger)
{
this.repository = repository;
this.azureProxy = azureProxy;
this.logger = logger;
this.isDisabled = configuration.GetValue("DisableBatchJobCleanup", false);
}
/// <summary>
/// Provides methods for blob storage access by using local path references in form of /storageaccount/container/blobpath
/// </summary>
/// <param name="logger">Logger <see cref="ILogger"/></param>
/// <param name="configuration">Configuration <see cref="IConfiguration"/></param>
/// <param name="azureProxy">Azure proxy <see cref="IAzureProxy"/></param>
public StorageAccessProvider(ILogger logger, IConfiguration configuration, IAzureProxy azureProxy)
{
this.logger = logger;
this.azureProxy = azureProxy;
this.defaultStorageAccountName = configuration["DefaultStorageAccountName"]; // This account contains the cromwell-executions container
logger.LogInformation($"DefaultStorageAccountName: {defaultStorageAccountName}");
externalStorageContainers = configuration["ExternalStorageContainers"]?.Split(new[] { ',', ';', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)
.Select(uri => {
if (StorageAccountUrlSegments.TryCreate(uri, out var s))
{
return(new ExternalStorageContainerInfo {
BlobEndpoint = s.BlobEndpoint, AccountName = s.AccountName, ContainerName = s.ContainerName, SasToken = s.SasToken
});
}
else
{
logger.LogError($"Invalid value '{uri}' found in 'ExternalStorageContainers' configuration. Value must be a valid azure storage account or container URL.");
return(null);
}
})
.Where(storageAccountInfo => storageAccountInfo is not null)
.ToList();
}
/// <summary>
/// Default constructor
/// </summary>
/// <param name="configuration">The configuration instance settings</param>
/// <param name="azureProxy">Azure Proxy</param>
/// <param name="repository">The main TES task database repository</param>
/// <param name="logger">The logger instance</param>
public DeleteCompletedBatchJobsHostedService(IConfiguration configuration, IAzureProxy azureProxy, IRepository <TesTask> repository, ILogger <DeleteCompletedBatchJobsHostedService> logger)
{
this.repository = repository;
this.azureProxy = azureProxy;
this.logger = logger;
isStopped = bool.TryParse(configuration["DisableBatchJobCleanup"], out var disableBatchJobCleanup) ? disableBatchJobCleanup : false;
}
/// <summary>
/// The constructor
/// </summary>
/// <param name="configuration"><see cref="IConfiguration"/></param>
/// <param name="azureProxy"><see cref="IAzureProxy"/></param>
/// <param name="storageAccessProvider"><see cref="IStorageAccessProvider"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
public ConfigurationUtils(IConfiguration configuration, IAzureProxy azureProxy, IStorageAccessProvider storageAccessProvider, ILogger logger)
{
this.configuration = configuration;
this.azureProxy = azureProxy;
this.storageAccessProvider = storageAccessProvider;
this.logger = logger;
}
private void CacheVmSizesAndPrices(IAzureProxy azureProxy)
{
try
{
var vms = azureProxy.GetVmSizesAndPricesAsync().Result;
logger.LogInformation($"Successfully retrieved info about {vms.Count} VMs.");
}
catch (Exception exc)
{
logger.LogError(exc, "Exception trying to GetVmSizesAndPrices on startup. Check if machine has Billing Reader writes on the subscription");
}
}
/// <summary>
/// Default constructor invoked by ASP.NET Core DI
/// </summary>
/// <param name="logger">Logger instance provided by ASP.NET Core DI</param>
/// <param name="configuration">Configuration</param>
/// <param name="azureProxy">Azure proxy</param>
public BatchScheduler(ILogger logger, IConfiguration configuration, IAzureProxy azureProxy)
{
this.logger = logger;
this.azureProxy = azureProxy;
defaultStorageAccountName = configuration["DefaultStorageAccountName"]; // This account contains the cromwell-executions container
usePreemptibleVmsOnly = bool.TryParse(configuration["UsePreemptibleVmsOnly"], out var temp) && temp;
externalStorageContainers = configuration["ExternalStorageContainers"]?.Split(new[] { ',', ';', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)
.Select(uri => {
if (StorageAccountUrlSegments.TryCreate(uri, out var s))
{
return(new ExternalStorageContainerInfo {
BlobEndpoint = s.BlobEndpoint, AccountName = s.AccountName, ContainerName = s.ContainerName, SasToken = s.SasToken
});
}
else
{
logger.LogError($"Invalid value '{uri}' found in 'ExternalStorageContainers' configuration. Value must be a valid azure storage account or container URL.");
return(null);
}
})
.Where(storageAccountInfo => storageAccountInfo != null)
.ToList();
logger.LogInformation($"DefaultStorageAccountName: {defaultStorageAccountName}");
logger.LogInformation($"usePreemptibleVmsOnly: {usePreemptibleVmsOnly}");
bool tesStateIsQueuedInitializingOrRunning(TesTask tesTask) => tesTask.State == TesState.QUEUEDEnum || tesTask.State == TesState.INITIALIZINGEnum || tesTask.State == TesState.RUNNINGEnum;
bool tesStateIsInitializingOrRunning(TesTask tesTask) => tesTask.State == TesState.INITIALIZINGEnum || tesTask.State == TesState.RUNNINGEnum;
bool tesStateIsQueuedOrInitializing(TesTask tesTask) => tesTask.State == TesState.QUEUEDEnum || tesTask.State == TesState.INITIALIZINGEnum;
tesTaskStateTransitions = new List <TesTaskStateTransition>()
{
new TesTaskStateTransition(tesTask => tesTask.State == TesState.CANCELEDEnum && tesTask.IsCancelRequested, batchTaskState: null, async tesTask => { await this.azureProxy.DeleteBatchJobAsync(tesTask.Id); tesTask.IsCancelRequested = false; }),
new TesTaskStateTransition(TesState.QUEUEDEnum, BatchTaskState.JobNotFound, tesTask => AddBatchJobAsync(tesTask)),
new TesTaskStateTransition(TesState.QUEUEDEnum, BatchTaskState.MissingBatchTask, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.QUEUEDEnum),
new TesTaskStateTransition(TesState.QUEUEDEnum, BatchTaskState.Initializing, TesState.INITIALIZINGEnum),
new TesTaskStateTransition(TesState.INITIALIZINGEnum, BatchTaskState.NodeAllocationFailed, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.QUEUEDEnum),
new TesTaskStateTransition(tesStateIsQueuedOrInitializing, BatchTaskState.Running, TesState.RUNNINGEnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.MoreThanOneActiveJobFound, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.SYSTEMERROREnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.CompletedSuccessfully, TesState.COMPLETEEnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.CompletedWithErrors, TesState.EXECUTORERROREnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.ActiveJobWithMissingAutoPool, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.QUEUEDEnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.NodeFailedDuringStartupOrExecution, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.EXECUTORERROREnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.NodeUnusable, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.EXECUTORERROREnum),
new TesTaskStateTransition(tesStateIsInitializingOrRunning, BatchTaskState.JobNotFound, TesState.SYSTEMERROREnum),
new TesTaskStateTransition(tesStateIsInitializingOrRunning, BatchTaskState.MissingBatchTask, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.SYSTEMERROREnum),
new TesTaskStateTransition(tesStateIsInitializingOrRunning, BatchTaskState.NodePreempted, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.QUEUEDEnum) // TODO: Implement preemption detection
};
}
/// <summary>
/// Default constructor invoked by ASP.NET Core DI
/// </summary>
/// <param name="logger">Logger instance provided by ASP.NET Core DI</param>
/// <param name="configuration">Configuration</param>
/// <param name="azureProxy">Azure proxy</param>
public BatchScheduler(ILogger logger, IConfiguration configuration, IAzureProxy azureProxy)
{
this.logger = logger;
this.azureProxy = azureProxy;
defaultStorageAccountName = configuration["DefaultStorageAccountName"]; // This account contains the cromwell-executions container
usePreemptibleVmsOnly = bool.TryParse(configuration["UsePreemptibleVmsOnly"], out var temp) ? temp : false;
externalStorageContainers = configuration["ExternalStorageContainers"]?.Split(new[] { ',', ';', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)
.SelectMany(e => externalStorageContainerRegex.Matches(e).Cast <Match>()
.Select(m => new ExternalStorageContainerInfo {
BlobEndpoint = m.Groups[1].Value, AccountName = m.Groups[2].Value, ContainerName = m.Groups[3].Value, SasToken = m.Groups[4].Value
}))
.ToList();
logger.LogInformation($"DefaultStorageAccountName: {defaultStorageAccountName}");
logger.LogInformation($"usePreemptibleVmsOnly: {usePreemptibleVmsOnly}");
bool tesStateIsQueuedInitializingOrRunning(TesTask tesTask) => tesTask.State == TesState.QUEUEDEnum || tesTask.State == TesState.INITIALIZINGEnum || tesTask.State == TesState.RUNNINGEnum;
bool tesStateIsInitializingOrRunning(TesTask tesTask) => tesTask.State == TesState.INITIALIZINGEnum || tesTask.State == TesState.RUNNINGEnum;
bool tesStateIsQueuedOrInitializing(TesTask tesTask) => tesTask.State == TesState.QUEUEDEnum || tesTask.State == TesState.INITIALIZINGEnum;
tesTaskStateTransitions = new List <TesTaskStateTransition>()
{
new TesTaskStateTransition(tesTask => tesTask.State == TesState.CANCELEDEnum && tesTask.IsCancelRequested, batchTaskState: null, async tesTask => { await this.azureProxy.DeleteBatchJobAsync(tesTask.Id); tesTask.IsCancelRequested = false; }),
new TesTaskStateTransition(TesState.QUEUEDEnum, BatchTaskState.JobNotFound, tesTask => AddBatchJobAsync(tesTask)),
new TesTaskStateTransition(TesState.QUEUEDEnum, BatchTaskState.MissingBatchTask, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.QUEUEDEnum),
new TesTaskStateTransition(TesState.QUEUEDEnum, BatchTaskState.Initializing, TesState.INITIALIZINGEnum),
new TesTaskStateTransition(TesState.INITIALIZINGEnum, BatchTaskState.NodeAllocationFailed, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.QUEUEDEnum),
new TesTaskStateTransition(tesStateIsQueuedOrInitializing, BatchTaskState.Running, TesState.RUNNINGEnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.MoreThanOneActiveJobFound, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.SYSTEMERROREnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.CompletedSuccessfully, TesState.COMPLETEEnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.CompletedWithErrors, TesState.EXECUTORERROREnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.PreparationTaskFailed, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.EXECUTORERROREnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.NodeDiskFull, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.EXECUTORERROREnum),
new TesTaskStateTransition(tesStateIsQueuedInitializingOrRunning, BatchTaskState.ActiveJobWithMissingAutoPool, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.QUEUEDEnum),
new TesTaskStateTransition(tesStateIsInitializingOrRunning, BatchTaskState.JobNotFound, TesState.SYSTEMERROREnum),
new TesTaskStateTransition(tesStateIsInitializingOrRunning, BatchTaskState.MissingBatchTask, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.SYSTEMERROREnum),
new TesTaskStateTransition(tesStateIsInitializingOrRunning, BatchTaskState.NodePreempted, tesTask => this.azureProxy.DeleteBatchJobAsync(tesTask.Id), TesState.QUEUEDEnum) // TODO: Implement preemption detection
};
}
/// <summary>
/// Contruct a <see cref="TaskServiceApiController"/>
/// </summary>
/// <param name="repository">The main <see cref="TesTask"/> database repository</param>
/// <param name="logger">The logger instance</param>
public TaskServiceApiController(IRepository <TesTask> repository, ILogger <TaskServiceApiController> logger, IAzureProxy azureProxy)
{
this.repository = repository;
this.logger = logger;
this.azureProxy = azureProxy;
}
/// <summary>
/// Default constructor
/// </summary>
/// <param name="azureProxy">Azure Proxy</param>
/// <param name="logger">The logger instance</param>
public DeleteOrphanedAutoPoolsHostedService(IAzureProxy azureProxy, ILogger <DeleteOrphanedAutoPoolsHostedService> logger)
{
this.azureProxy = azureProxy;
this.logger = logger;
}
/// <summary>
/// Contructor to create a cache of <see cref="IAzureProxy"/>
/// </summary>
/// <param name="azureProxy"><see cref="AzureProxy"/></param>
/// <param name="cache">Lazy cache using <see cref="IAppCache"/></param>
/// <param name="logger"><see cref="ILogger"/> instance</param>
public CachingAzureProxy(IAzureProxy azureProxy, IAppCache cache, ILogger <CachingAzureProxy> logger)
{
this.azureProxy = azureProxy;
this.cache = cache;
this.logger = logger;
}
private void ConfigureServices(IServiceCollection services, IAppCache cache, AzureProxy azureProxy, IAzureProxy cachingAzureProxy, IStorageAccessProvider storageAccessProvider, IRepository <TesTask> repository)
=> services.AddSingleton(cache)
.AddSingleton(cachingAzureProxy)
.AddSingleton(azureProxy)
.AddControllers()
.AddNewtonsoftJson(opts =>
{
opts.SerializerSettings.ContractResolver = new CamelCasePropertyNamesContractResolver();
opts.SerializerSettings.Converters.Add(new StringEnumConverter(new CamelCaseNamingStrategy()));
}).Services
.AddSingleton <IRepository <TesTask> >(new CachingWithRetriesRepository <TesTask>(repository))
.AddSingleton <IBatchScheduler>(new BatchScheduler(loggerFactory.CreateLogger <BatchScheduler>(), Configuration, cachingAzureProxy, storageAccessProvider))
.AddSwaggerGen(c =>
{
c.SwaggerDoc("0.3.0", new OpenApiInfo
{
Version = "0.3.0",
Title = "Task Execution Service",
Description = "Task Execution Service (ASP.NET Core 3.1)",
Contact = new OpenApiContact()
{
Name = "Microsoft Genomics",
Url = new Uri("https://github.com/microsoft/CromwellOnAzure")
},
});
c.CustomSchemaIds(type => type.FullName);
c.IncludeXmlComments($"{AppContext.BaseDirectory}{Path.DirectorySeparatorChar}{Assembly.GetEntryAssembly().GetName().Name}.xml");
c.OperationFilter <GeneratePathParamsValidationFilter>();
})
.AddHostedService <Scheduler>()
.AddHostedService <DeleteCompletedBatchJobsHostedService>()
.AddHostedService <DeleteOrphanedBatchJobsHostedService>()
.AddHostedService <DeleteOrphanedAutoPoolsHostedService>()
.AddHostedService <RefreshVMSizesAndPricesHostedService>()
// Configure AppInsights Azure Service when in PRODUCTION environment
.IfThenElse(hostingEnvironment.IsProduction(),
s =>
{
var applicationInsightsAccountName = Configuration["ApplicationInsightsAccountName"];
var instrumentationKey = AzureProxy.GetAppInsightsInstrumentationKeyAsync(applicationInsightsAccountName).Result;
if (instrumentationKey is not null)
{
return(s.AddApplicationInsightsTelemetry(instrumentationKey));
}
return(s);
},
s => s.AddApplicationInsightsTelemetry());
private static TaskServiceApiController GetTaskServiceApiController(IRepository <TesTask> repository, IAzureProxy azureProxy) => new(repository, new NullLogger <TaskServiceApiController>(), azureProxy);
private static TaskServiceApiController GetTaskServiceApiController(IAzureProxy azureProxy) => GetTaskServiceApiController(new Mock <IRepository <TesTask> >().Object, azureProxy);
/// <summary>
/// Contructor to create a cache of <see cref="IAzureProxy"/>
/// </summary>
/// <param name="azureProxy"><see cref="AzureProxy"/></param>
/// <param name="cache">Lazy cache using <see cref="IAppCache"/></param>
public CachingWithRetriesAzureProxy(IAzureProxy azureProxy, IAppCache cache)
{
this.azureProxy = azureProxy;
this.cache = cache;
}
private TaskServiceApiController GetTaskServiceApiController(IRepository <TesTask> repository, IAzureProxy azureProxy)
{
return(new TaskServiceApiController(repository, new NullLogger <TaskServiceApiController>(), azureProxy));
}
/// <summary>
/// Default constructor
/// </summary>
/// <param name="azureProxy">Azure Proxy</param>
/// <param name="repository">The main TES task database repository</param>
/// <param name="logger">The logger instance</param>
public DeleteOrphanedBatchJobsHostedService(IAzureProxy azureProxy, IRepository <TesTask> repository, ILogger <DeleteOrphanedBatchJobsHostedService> logger)
{
this.repository = repository;
this.azureProxy = azureProxy;
this.logger = logger;
}
