public async Task CleanUpAsync(PerProcessContext context)
{
var blobClient = BlobStorageUtilities.GetBlobClient(context.Global);
// Delete catalog2dnx artifacts.
await CleanUpUtilities.DeleteBlobsWithPrefix(
blobClient,
context.Global.FlatContainerContainerName,
context.FlatContainerStoragePath,
_logger);
}
public async Task ProcessAsync(PerProcessContext context)
{
var evenCount = context.MessageCount / context.WorkerCount;
var extraCount = evenCount + (context.MessageCount % context.WorkerCount);
var tasks = Enumerable
.Range(0, context.WorkerCount)
.Select(x => ProcessPerWorkerAsync(context, x == 0 ? extraCount : evenCount))
.ToList();
await Task.WhenAll(tasks);
}
private static IServiceProvider InitializeAndGetServiceProvider()
{
ServicePointManager.DefaultConnectionLimit = 64;
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
var configurationRoot = GetConfigurationRoot();
var instrumentationKey = configurationRoot
.GetSection(ConfigurationSectionName)
.GetValue <string>(nameof(V3PerPackageConfiguration.InstrumentationKey));
ApplicationInsights.Initialize(instrumentationKey);
var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true);
var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Information);
var serviceCollection = new ServiceCollection();
serviceCollection.Configure <V3PerPackageConfiguration>(configurationRoot.GetSection(ConfigurationSectionName));
serviceCollection.AddOptions();
serviceCollection.Add(ServiceDescriptor.Scoped(typeof(IOptionsSnapshot <>), typeof(NonCachingOptionsSnapshot <>)));
serviceCollection.AddLogging();
serviceCollection.AddSingleton(loggerFactory);
serviceCollection.AddSingleton(new ControlledDisposeHttpClientHandler());
serviceCollection.AddTransient <HttpMessageHandler>(x => x.GetRequiredService <ControlledDisposeHttpClientHandler>());
serviceCollection.AddSingleton(x => new HttpClient(x.GetRequiredService <HttpMessageHandler>()));
serviceCollection.AddTransient <Func <HttpMessageHandler> >(x => () => x.GetRequiredService <HttpMessageHandler>());
serviceCollection.AddTransient <PerBatchProcessor>();
serviceCollection.AddTransient <ITelemetryService, TelemetryService>();
serviceCollection.AddTransient <PerWorkerProcessor>();
serviceCollection.AddTransient <PerProcessProcessor>();
serviceCollection.AddSingleton <StringLocker>();
serviceCollection.AddTransient <EnqueueCollector>();
serviceCollection.AddTransient <EnqueueCommand>();
serviceCollection.AddTransient <CleanUpCommand>();
serviceCollection.AddSingleton(x =>
{
var globalContext = x.GetRequiredService <GlobalContext>();
var perProcessContext = new PerProcessContext(
globalContext,
UniqueName.New("process"),
WorkerCount,
MessageCount,
BatchSize);
var blobClient = BlobStorageUtilities.GetBlobClient(globalContext);
var flatContainerUrl = $"{blobClient.BaseUri.AbsoluteUri}/{globalContext.FlatContainerContainerName}/{perProcessContext.Name}";
RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerUrl);
return(perProcessContext);
});
serviceCollection.AddTransient(x =>
{
var settings = x.GetRequiredService <IOptionsSnapshot <V3PerPackageConfiguration> >();
return(new GlobalContext(
settings.Value.StorageBaseAddress,
settings.Value.StorageAccountName,
settings.Value.StorageKeyValue,
settings.Value.ContentBaseAddress,
settings.Value.GalleryBaseAddress));
});
serviceCollection.AddSingleton(new TelemetryClient());
serviceCollection.AddTransient <IStorageQueue <PackageMessage> >(x =>
{
var globalContext = x.GetRequiredService <GlobalContext>();
var storageCredentials = new StorageCredentials(globalContext.StorageAccountName, globalContext.StorageKeyValue);
var storageAccount = new CloudStorageAccount(storageCredentials, useHttps: true);
return(new StorageQueue <PackageMessage>(
new AzureStorageQueue(storageAccount, "v3perpackage"),
new JsonMessageSerializer <PackageMessage>(JsonSerializerUtility.SerializerSettings),
PackageMessage.Version));
});
return(serviceCollection.BuildServiceProvider());
}
private async Task ProcessPerWorkerAsync(PerProcessContext processContext, int messageCount)
{
var context = new PerWorkerContext(processContext, UniqueName.New("worker"));
await _perWorkerProcessor.ProcessAsync(context, messageCount);
}