private async Task <bool> ProcessNextMessageAsync(PerWorkerContext workerContext, int batchSize)
{
var batchContext = new PerBatchContext(workerContext, UniqueName.New("batch"));
var packageIdentities = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var messages = new List <StorageQueueMessage <PackageMessage> >();
var packageContexts = new List <PerPackageContext>();
StorageQueueMessage <PackageMessage> lastMessage;
do
{
lastMessage = await _queue.GetNextAsync(CancellationToken.None);
if (lastMessage != null)
{
var packageId = lastMessage.Contents.PackageId.Trim();
var packageVersion = NuGetVersion.Parse(lastMessage.Contents.PackageVersion.Trim()).ToNormalizedString();
var packageIdentity = $"{packageId}/{packageVersion}";
// If this is a duplicate package, complete it and skip it.
if (!packageIdentities.Add(packageIdentity))
{
await _queue.RemoveAsync(lastMessage, CancellationToken.None);
continue;
}
messages.Add(lastMessage);
packageContexts.Add(new PerPackageContext(batchContext, packageId, packageVersion));
}
}while (messages.Count < batchSize && lastMessage != null && lastMessage.DequeueCount < 10);
if (packageContexts.Count == 0)
{
return(false);
}
var complete = await ProcessPackagesAsync(batchContext, packageContexts);
if (complete)
{
foreach (var message in messages)
{
try
{
await _queue.RemoveAsync(message, CancellationToken.None);
}
catch (StorageException ex) when(ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
// Ignore this error. The message has already been removed.
}
}
}
return(true);
}
private static IServiceProvider InitializeAndGetServiceProvider()
{
ServicePointManager.DefaultConnectionLimit = 64;
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
var configurationRoot = GetConfigurationRoot();
var instrumentationKey = configurationRoot
.GetSection(ConfigurationSectionName)
.GetValue <string>(nameof(V3PerPackageConfiguration.InstrumentationKey));
ApplicationInsights.Initialize(instrumentationKey);
var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true);
var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Information);
var serviceCollection = new ServiceCollection();
serviceCollection.Configure <V3PerPackageConfiguration>(configurationRoot.GetSection(ConfigurationSectionName));
serviceCollection.AddOptions();
serviceCollection.Add(ServiceDescriptor.Scoped(typeof(IOptionsSnapshot <>), typeof(NonCachingOptionsSnapshot <>)));
serviceCollection.AddLogging();
serviceCollection.AddSingleton(loggerFactory);
serviceCollection.AddSingleton(new ControlledDisposeHttpClientHandler());
serviceCollection.AddTransient <HttpMessageHandler>(x => x.GetRequiredService <ControlledDisposeHttpClientHandler>());
serviceCollection.AddSingleton(x => new HttpClient(x.GetRequiredService <HttpMessageHandler>()));
serviceCollection.AddTransient <Func <HttpMessageHandler> >(x => () => x.GetRequiredService <HttpMessageHandler>());
serviceCollection.AddTransient <PerBatchProcessor>();
serviceCollection.AddTransient <ITelemetryService, TelemetryService>();
serviceCollection.AddTransient <PerWorkerProcessor>();
serviceCollection.AddTransient <PerProcessProcessor>();
serviceCollection.AddSingleton <StringLocker>();
serviceCollection.AddTransient <EnqueueCollector>();
serviceCollection.AddTransient <EnqueueCommand>();
serviceCollection.AddTransient <CleanUpCommand>();
serviceCollection.AddSingleton(x =>
{
var globalContext = x.GetRequiredService <GlobalContext>();
var perProcessContext = new PerProcessContext(
globalContext,
UniqueName.New("process"),
WorkerCount,
MessageCount,
BatchSize);
var blobClient = BlobStorageUtilities.GetBlobClient(globalContext);
var flatContainerUrl = $"{blobClient.BaseUri.AbsoluteUri}/{globalContext.FlatContainerContainerName}/{perProcessContext.Name}";
RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerUrl);
return(perProcessContext);
});
serviceCollection.AddTransient(x =>
{
var settings = x.GetRequiredService <IOptionsSnapshot <V3PerPackageConfiguration> >();
return(new GlobalContext(
settings.Value.StorageBaseAddress,
settings.Value.StorageAccountName,
settings.Value.StorageKeyValue,
settings.Value.ContentBaseAddress,
settings.Value.GalleryBaseAddress));
});
serviceCollection.AddSingleton(new TelemetryClient());
serviceCollection.AddTransient <IStorageQueue <PackageMessage> >(x =>
{
var globalContext = x.GetRequiredService <GlobalContext>();
var storageCredentials = new StorageCredentials(globalContext.StorageAccountName, globalContext.StorageKeyValue);
var storageAccount = new CloudStorageAccount(storageCredentials, useHttps: true);
return(new StorageQueue <PackageMessage>(
new AzureStorageQueue(storageAccount, "v3perpackage"),
new JsonMessageSerializer <PackageMessage>(JsonSerializerUtility.SerializerSettings),
PackageMessage.Version));
});
return(serviceCollection.BuildServiceProvider());
}
private async Task ProcessPerWorkerAsync(PerProcessContext processContext, int messageCount)
{
var context = new PerWorkerContext(processContext, UniqueName.New("worker"));
await _perWorkerProcessor.ProcessAsync(context, messageCount);
}