public async Task CleanUpAsync(PerProcessContext context) { var blobClient = BlobStorageUtilities.GetBlobClient(context.Global); // Delete catalog2dnx artifacts. await CleanUpUtilities.DeleteBlobsWithPrefix( blobClient, context.Global.FlatContainerContainerName, context.FlatContainerStoragePath, _logger); }
public async Task ProcessAsync(PerProcessContext context) { var evenCount = context.MessageCount / context.WorkerCount; var extraCount = evenCount + (context.MessageCount % context.WorkerCount); var tasks = Enumerable .Range(0, context.WorkerCount) .Select(x => ProcessPerWorkerAsync(context, x == 0 ? extraCount : evenCount)) .ToList(); await Task.WhenAll(tasks); }
private static IServiceProvider InitializeAndGetServiceProvider() { ServicePointManager.DefaultConnectionLimit = 64; ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3; ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; var configurationRoot = GetConfigurationRoot(); var instrumentationKey = configurationRoot .GetSection(ConfigurationSectionName) .GetValue <string>(nameof(V3PerPackageConfiguration.InstrumentationKey)); ApplicationInsights.Initialize(instrumentationKey); var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true); var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Information); var serviceCollection = new ServiceCollection(); serviceCollection.Configure <V3PerPackageConfiguration>(configurationRoot.GetSection(ConfigurationSectionName)); serviceCollection.AddOptions(); serviceCollection.Add(ServiceDescriptor.Scoped(typeof(IOptionsSnapshot <>), typeof(NonCachingOptionsSnapshot <>))); serviceCollection.AddLogging(); serviceCollection.AddSingleton(loggerFactory); serviceCollection.AddSingleton(new ControlledDisposeHttpClientHandler()); serviceCollection.AddTransient <HttpMessageHandler>(x => x.GetRequiredService <ControlledDisposeHttpClientHandler>()); serviceCollection.AddSingleton(x => new HttpClient(x.GetRequiredService <HttpMessageHandler>())); serviceCollection.AddTransient <Func <HttpMessageHandler> >(x => () => x.GetRequiredService <HttpMessageHandler>()); serviceCollection.AddTransient <PerBatchProcessor>(); serviceCollection.AddTransient <ITelemetryService, TelemetryService>(); serviceCollection.AddTransient <PerWorkerProcessor>(); serviceCollection.AddTransient <PerProcessProcessor>(); serviceCollection.AddSingleton <StringLocker>(); serviceCollection.AddTransient <EnqueueCollector>(); serviceCollection.AddTransient <EnqueueCommand>(); serviceCollection.AddTransient <CleanUpCommand>(); serviceCollection.AddSingleton(x => { var globalContext = x.GetRequiredService <GlobalContext>(); var perProcessContext = new PerProcessContext( globalContext, UniqueName.New("process"), WorkerCount, MessageCount, BatchSize); var blobClient = BlobStorageUtilities.GetBlobClient(globalContext); var flatContainerUrl = $"{blobClient.BaseUri.AbsoluteUri}/{globalContext.FlatContainerContainerName}/{perProcessContext.Name}"; RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerUrl); return(perProcessContext); }); serviceCollection.AddTransient(x => { var settings = x.GetRequiredService <IOptionsSnapshot <V3PerPackageConfiguration> >(); return(new GlobalContext( settings.Value.StorageBaseAddress, settings.Value.StorageAccountName, settings.Value.StorageKeyValue, settings.Value.ContentBaseAddress, settings.Value.GalleryBaseAddress)); }); serviceCollection.AddSingleton(new TelemetryClient()); serviceCollection.AddTransient <IStorageQueue <PackageMessage> >(x => { var globalContext = x.GetRequiredService <GlobalContext>(); var storageCredentials = new StorageCredentials(globalContext.StorageAccountName, globalContext.StorageKeyValue); var storageAccount = new CloudStorageAccount(storageCredentials, useHttps: true); return(new StorageQueue <PackageMessage>( new AzureStorageQueue(storageAccount, "v3perpackage"), new JsonMessageSerializer <PackageMessage>(JsonSerializerUtility.SerializerSettings), PackageMessage.Version)); }); return(serviceCollection.BuildServiceProvider()); }
private async Task ProcessPerWorkerAsync(PerProcessContext processContext, int messageCount) { var context = new PerWorkerContext(processContext, UniqueName.New("worker")); await _perWorkerProcessor.ProcessAsync(context, messageCount); }