private Collection <TKey, TValue> CreateCollection <TKey, TValue>(string collectionName) { var storage = _storageFactory.Create(collectionName + ".dat"); var history = new History <TKey, TValue>(storage); return(new Collection <TKey, TValue>(history)); }
private static KeyValuePair <string, DurableCursor> GetDurableCursor(IStorageFactory storageFactory, string name) { var cursorStorage = storageFactory.Create(); var cursorUri = cursorStorage.ResolveUri(name); return(new KeyValuePair <string, DurableCursor>( cursorUri.AbsoluteUri, new DurableCursor(cursorUri, cursorStorage, DateTime.MinValue))); }
public static async Task <HttpResponseMessage> Run( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] GridEvent <JObject>[] gridEvents, [Table(nameof(Collections.Events), Connection = AppSettingContainingConnectionString)] CloudTable eventsTable, [Inject(typeof(IStorageFactory <EventHistoryTableEntity>))] IStorageFactory <EventHistoryTableEntity> storageFactory, ILogger logger) { var eventProcessor = new EventHistoryProcessor(gridEvents, storageFactory.Create(eventsTable), logger); return(await ProcessorRunner.Run(eventProcessor)); }
public static async Task <HttpResponseMessage> Run( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] GridEvent <StorageEvent>[] gridEvents, [Table(nameof(Collections.AccountsV1), Connection = AppSettingContainingConnectionString)] CloudTable accountsV1, [Table(nameof(Collections.UsersV1), Connection = AppSettingContainingConnectionString)] CloudTable UsersV1, [Table(nameof(Collections.ContactsV1), Connection = AppSettingContainingConnectionString)] CloudTable contactsV1, [Inject(typeof(IStorageFactory <StorageEventTableEntity>))] IStorageFactory <StorageEventTableEntity> storageFactory, ILogger logger) { var storageTables = new Dictionary <string, IStorage <StorageEventTableEntity> > { { Collections.AccountsV1, storageFactory.Create(accountsV1) }, { Collections.UsersV1, storageFactory.Create(UsersV1) }, { Collections.ContactsV1, storageFactory.Create(contactsV1) }, }; var eventProcessor = new StorageEventProcessor(gridEvents, storageTables, logger); return(await ProcessorRunner.Run(eventProcessor)); }
private async Task ExecuteAsync(CancellationToken token) { using (var cancelledCts = new CancellationTokenSource()) using (var produceWorkCts = new CancellationTokenSource()) { // Initialize the indexes, container and excluded packages data. await InitializeAsync(); // Here, we fetch the current catalog timestamp to use as the initial cursor value for // catalog2azuresearch. The idea here is that database is always more up-to-date than the catalog. // We're about to read the database so if we capture a catalog timestamp now, we are guaranteed that // any data we get from a database query will be more recent than the data represented by this catalog // timestamp. When catalog2azuresearch starts up for the first time to update the index produced by this // job, it will probably encounter some duplicate packages, but this is okay. // // Note that we could capture any dependency cursors here instead of catalog cursor, but this is // pointless because there is no reliable way to filter out data fetched from the database based on a // catalog-based cursor value. Suppose the dependency cursor is catalog2registration. If // catalog2registration is very behind, then the index produced by this job will include packages that // are not yet restorable (since they are not in the registration hives). This could lead to a case // where a user is able to search for a package that he cannot restore. We mitigate this risk by // trusting that our end-to-end tests will fail when catalog2registration (or any other V3 component) is // broken, this blocking the deployment of new Azure Search indexes. var catalogIndex = await _catalogClient.GetIndexAsync(_options.Value.CatalogIndexUrl); var initialCursorValue = catalogIndex.CommitTimestamp; _logger.LogInformation("The initial cursor value will be {CursorValue:O}.", initialCursorValue); var initialAuxiliaryData = await PushAllPackageRegistrationsAsync(cancelledCts, produceWorkCts); // Write the owner data file. await WriteOwnerDataAsync(initialAuxiliaryData.Owners); // Write the download data file. await WriteDownloadDataAsync(initialAuxiliaryData.Downloads); // Write the verified packages data file. await WriteVerifiedPackagesDataAsync(initialAuxiliaryData.VerifiedPackages); // Write popularity transfers data file. await WritePopularityTransfersDataAsync(initialAuxiliaryData.PopularityTransfers); // Write the cursor. _logger.LogInformation("Writing the initial cursor value to be {CursorValue:O}.", initialCursorValue); var frontCursorStorage = _storageFactory.Create(); var frontCursor = new DurableCursor( frontCursorStorage.ResolveUri(Catalog2AzureSearchCommand.CursorRelativeUri), frontCursorStorage, DateTime.MinValue); frontCursor.Value = initialCursorValue.UtcDateTime; await frontCursor.SaveAsync(token); } }
private async Task ExecuteAsync(CancellationToken token) { // Initialize the cursors. ReadCursor backCursor; if (_options.Value.DependencyCursorUrls != null && _options.Value.DependencyCursorUrls.Any()) { _logger.LogInformation("Depending on cursors: {DependencyCursorUrls}", _options.Value.DependencyCursorUrls); backCursor = new AggregateCursor(_options .Value .DependencyCursorUrls.Select(r => new HttpReadCursor(new Uri(r), _handlerFunc))); } else { _logger.LogInformation("Depending on no cursors, meaning the job will process up to the latest catalog information."); backCursor = MemoryCursor.CreateMax(); } var frontCursorStorage = _storageFactory.Create(); var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri); var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue); _logger.LogInformation("Using cursor: {CursurUrl}", frontCursorUri.AbsoluteUri); LogContainerUrl(HiveType.Legacy, c => c.LegacyStorageContainer); LogContainerUrl(HiveType.Gzipped, c => c.GzippedStorageContainer); LogContainerUrl(HiveType.SemVer2, c => c.SemVer2StorageContainer); // Optionally create the containers. if (_options.Value.CreateContainers) { await CreateContainerIfNotExistsAsync(c => c.LegacyStorageContainer); await CreateContainerIfNotExistsAsync(c => c.GzippedStorageContainer); await CreateContainerIfNotExistsAsync(c => c.SemVer2StorageContainer); } await frontCursor.LoadAsync(token); await backCursor.LoadAsync(token); _logger.LogInformation( "The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.", frontCursor.Value, backCursor.Value); // Run the collector. await _collector.RunAsync( frontCursor, backCursor, token); }
private async Task ProcessIconsAsync( ConcurrentBag <IReadOnlyCollection <CatalogCommitItem> > items, CancellationToken cancellationToken) { await Task.Yield(); var targetStorage = _targetStorageFactory.Create(); var iconCacheStorage = _iconCacheStorageFactory.Create(); using (_logger.BeginScope("{CallGuid}", Guid.NewGuid())) while (items.TryTake(out var entries)) { var firstItem = entries.First(); using (_logger.BeginScope("Processing commits for {PackageId} {PackageVersion}", firstItem.PackageIdentity.Id, firstItem.PackageIdentity.Version)) { foreach (var item in entries) { if (item.IsPackageDetails) { PackageDetailsCatalogLeaf leaf; try { leaf = await _catalogClient.GetPackageDetailsLeafAsync(item.Uri.AbsoluteUri); } catch (Exception e) { _logger.LogError(0, e, "Error while trying to retrieve catalog leaf {LeafUrl}", item.Uri.AbsoluteUri); throw; } await _catalogLeafDataProcessor.ProcessPackageDetailsLeafAsync(targetStorage, iconCacheStorage, item, leaf.IconUrl, leaf.IconFile, cancellationToken); } else if (item.IsPackageDelete) { await _catalogLeafDataProcessor.ProcessPackageDeleteLeafAsync(targetStorage, item, cancellationToken); } } } } }
public void SetUp() { _sut = _factory.Create <Crate>(); }
private CatalogStorage GetStorage(string stateString) { return(_storageFactory.Create(stateString.ToLowerInvariant())); }
public string GetConfig() { var factory = _storageFactory.Create("config"); return(factory.Deserialize()); }
public Storage Create(string name = null) { return(_innerStorageFactory.Create($"{_name}/{name}")); }
private async Task ExecuteAsync(CancellationToken token) { // Initialize the cursors. ReadCursor backCursor; if (_options.Value.DependencyCursorUrls != null && _options.Value.DependencyCursorUrls.Any()) { _logger.LogInformation("Depending on cursors:{DependencyCursorUrls}", _options.Value.DependencyCursorUrls); backCursor = new AggregateCursor(_options .Value .DependencyCursorUrls.Select(r => new HttpReadCursor(new Uri(r), _handlerFunc))); } else { _logger.LogInformation("Depending on no cursors, meaning the job will process up to the latest catalog information."); backCursor = MemoryCursor.CreateMax(); } var frontCursorStorage = _storageFactory.Create(); var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri); var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue); // Log information about where state will be kept. _logger.LogInformation( "Using storage URL: {ContainerUrl}/{StoragePath}", CloudStorageAccount.Parse(_options.Value.StorageConnectionString) .CreateCloudBlobClient() .GetContainerReference(_options.Value.StorageContainer) .Uri .AbsoluteUri, _options.Value.NormalizeStoragePath()); _logger.LogInformation("Using cursor: {CursurUrl}", frontCursorUri.AbsoluteUri); _logger.LogInformation("Using search service: {SearchServiceName}", _options.Value.SearchServiceName); _logger.LogInformation("Using search index: {IndexName}", _options.Value.SearchIndexName); _logger.LogInformation("Using hijack index: {IndexName}", _options.Value.HijackIndexName); // Optionally create the indexes. if (_options.Value.CreateContainersAndIndexes) { await _blobContainerBuilder.CreateIfNotExistsAsync(); await _indexBuilder.CreateSearchIndexIfNotExistsAsync(); await _indexBuilder.CreateHijackIndexIfNotExistsAsync(); } await frontCursor.LoadAsync(token); await backCursor.LoadAsync(token); _logger.LogInformation( "The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.", frontCursor.Value, backCursor.Value); // Run the collector. await _collector.RunAsync( frontCursor, backCursor, token); }
public string GetData() => _storageFactory.Create().Deserialize();