private Collection <TKey, TValue> CreateCollection <TKey, TValue>(string collectionName)
{
var storage = _storageFactory.Create(collectionName + ".dat");
var history = new History <TKey, TValue>(storage);
return(new Collection <TKey, TValue>(history));
}
private static KeyValuePair <string, DurableCursor> GetDurableCursor(IStorageFactory storageFactory, string name)
{
var cursorStorage = storageFactory.Create();
var cursorUri = cursorStorage.ResolveUri(name);
return(new KeyValuePair <string, DurableCursor>(
cursorUri.AbsoluteUri,
new DurableCursor(cursorUri, cursorStorage, DateTime.MinValue)));
}
public static async Task <HttpResponseMessage> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] GridEvent <JObject>[] gridEvents,
[Table(nameof(Collections.Events), Connection = AppSettingContainingConnectionString)] CloudTable eventsTable,
[Inject(typeof(IStorageFactory <EventHistoryTableEntity>))] IStorageFactory <EventHistoryTableEntity> storageFactory,
ILogger logger)
{
var eventProcessor = new EventHistoryProcessor(gridEvents, storageFactory.Create(eventsTable), logger);
return(await ProcessorRunner.Run(eventProcessor));
}
public static async Task <HttpResponseMessage> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] GridEvent <StorageEvent>[] gridEvents,
[Table(nameof(Collections.AccountsV1), Connection = AppSettingContainingConnectionString)] CloudTable accountsV1,
[Table(nameof(Collections.UsersV1), Connection = AppSettingContainingConnectionString)] CloudTable UsersV1,
[Table(nameof(Collections.ContactsV1), Connection = AppSettingContainingConnectionString)] CloudTable contactsV1,
[Inject(typeof(IStorageFactory <StorageEventTableEntity>))] IStorageFactory <StorageEventTableEntity> storageFactory,
ILogger logger)
{
var storageTables = new Dictionary <string, IStorage <StorageEventTableEntity> >
{
{ Collections.AccountsV1, storageFactory.Create(accountsV1) },
{ Collections.UsersV1, storageFactory.Create(UsersV1) },
{ Collections.ContactsV1, storageFactory.Create(contactsV1) },
};
var eventProcessor = new StorageEventProcessor(gridEvents, storageTables, logger);
return(await ProcessorRunner.Run(eventProcessor));
}
private async Task ExecuteAsync(CancellationToken token)
{
using (var cancelledCts = new CancellationTokenSource())
using (var produceWorkCts = new CancellationTokenSource())
{
// Initialize the indexes, container and excluded packages data.
await InitializeAsync();
// Here, we fetch the current catalog timestamp to use as the initial cursor value for
// catalog2azuresearch. The idea here is that database is always more up-to-date than the catalog.
// We're about to read the database so if we capture a catalog timestamp now, we are guaranteed that
// any data we get from a database query will be more recent than the data represented by this catalog
// timestamp. When catalog2azuresearch starts up for the first time to update the index produced by this
// job, it will probably encounter some duplicate packages, but this is okay.
//
// Note that we could capture any dependency cursors here instead of catalog cursor, but this is
// pointless because there is no reliable way to filter out data fetched from the database based on a
// catalog-based cursor value. Suppose the dependency cursor is catalog2registration. If
// catalog2registration is very behind, then the index produced by this job will include packages that
// are not yet restorable (since they are not in the registration hives). This could lead to a case
// where a user is able to search for a package that he cannot restore. We mitigate this risk by
// trusting that our end-to-end tests will fail when catalog2registration (or any other V3 component) is
// broken, this blocking the deployment of new Azure Search indexes.
var catalogIndex = await _catalogClient.GetIndexAsync(_options.Value.CatalogIndexUrl);
var initialCursorValue = catalogIndex.CommitTimestamp;
_logger.LogInformation("The initial cursor value will be {CursorValue:O}.", initialCursorValue);
var initialAuxiliaryData = await PushAllPackageRegistrationsAsync(cancelledCts, produceWorkCts);
// Write the owner data file.
await WriteOwnerDataAsync(initialAuxiliaryData.Owners);
// Write the download data file.
await WriteDownloadDataAsync(initialAuxiliaryData.Downloads);
// Write the verified packages data file.
await WriteVerifiedPackagesDataAsync(initialAuxiliaryData.VerifiedPackages);
// Write popularity transfers data file.
await WritePopularityTransfersDataAsync(initialAuxiliaryData.PopularityTransfers);
// Write the cursor.
_logger.LogInformation("Writing the initial cursor value to be {CursorValue:O}.", initialCursorValue);
var frontCursorStorage = _storageFactory.Create();
var frontCursor = new DurableCursor(
frontCursorStorage.ResolveUri(Catalog2AzureSearchCommand.CursorRelativeUri),
frontCursorStorage,
DateTime.MinValue);
frontCursor.Value = initialCursorValue.UtcDateTime;
await frontCursor.SaveAsync(token);
}
}
private async Task ExecuteAsync(CancellationToken token)
{
// Initialize the cursors.
ReadCursor backCursor;
if (_options.Value.DependencyCursorUrls != null &&
_options.Value.DependencyCursorUrls.Any())
{
_logger.LogInformation("Depending on cursors: {DependencyCursorUrls}", _options.Value.DependencyCursorUrls);
backCursor = new AggregateCursor(_options
.Value
.DependencyCursorUrls.Select(r => new HttpReadCursor(new Uri(r), _handlerFunc)));
}
else
{
_logger.LogInformation("Depending on no cursors, meaning the job will process up to the latest catalog information.");
backCursor = MemoryCursor.CreateMax();
}
var frontCursorStorage = _storageFactory.Create();
var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri);
var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue);
_logger.LogInformation("Using cursor: {CursurUrl}", frontCursorUri.AbsoluteUri);
LogContainerUrl(HiveType.Legacy, c => c.LegacyStorageContainer);
LogContainerUrl(HiveType.Gzipped, c => c.GzippedStorageContainer);
LogContainerUrl(HiveType.SemVer2, c => c.SemVer2StorageContainer);
// Optionally create the containers.
if (_options.Value.CreateContainers)
{
await CreateContainerIfNotExistsAsync(c => c.LegacyStorageContainer);
await CreateContainerIfNotExistsAsync(c => c.GzippedStorageContainer);
await CreateContainerIfNotExistsAsync(c => c.SemVer2StorageContainer);
}
await frontCursor.LoadAsync(token);
await backCursor.LoadAsync(token);
_logger.LogInformation(
"The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.",
frontCursor.Value,
backCursor.Value);
// Run the collector.
await _collector.RunAsync(
frontCursor,
backCursor,
token);
}
private async Task ProcessIconsAsync(
ConcurrentBag <IReadOnlyCollection <CatalogCommitItem> > items,
CancellationToken cancellationToken)
{
await Task.Yield();
var targetStorage = _targetStorageFactory.Create();
var iconCacheStorage = _iconCacheStorageFactory.Create();
using (_logger.BeginScope("{CallGuid}", Guid.NewGuid()))
while (items.TryTake(out var entries))
{
var firstItem = entries.First();
using (_logger.BeginScope("Processing commits for {PackageId} {PackageVersion}", firstItem.PackageIdentity.Id, firstItem.PackageIdentity.Version))
{
foreach (var item in entries)
{
if (item.IsPackageDetails)
{
PackageDetailsCatalogLeaf leaf;
try
{
leaf = await _catalogClient.GetPackageDetailsLeafAsync(item.Uri.AbsoluteUri);
}
catch (Exception e)
{
_logger.LogError(0, e, "Error while trying to retrieve catalog leaf {LeafUrl}", item.Uri.AbsoluteUri);
throw;
}
await _catalogLeafDataProcessor.ProcessPackageDetailsLeafAsync(targetStorage, iconCacheStorage, item, leaf.IconUrl, leaf.IconFile, cancellationToken);
}
else if (item.IsPackageDelete)
{
await _catalogLeafDataProcessor.ProcessPackageDeleteLeafAsync(targetStorage, item, cancellationToken);
}
}
}
}
}
public void SetUp()
{
_sut = _factory.Create <Crate>();
}
private CatalogStorage GetStorage(string stateString)
{
return(_storageFactory.Create(stateString.ToLowerInvariant()));
}
public string GetConfig()
{
var factory = _storageFactory.Create("config");
return(factory.Deserialize());
}
public Storage Create(string name = null)
{
return(_innerStorageFactory.Create($"{_name}/{name}"));
}
private async Task ExecuteAsync(CancellationToken token)
{
// Initialize the cursors.
ReadCursor backCursor;
if (_options.Value.DependencyCursorUrls != null &&
_options.Value.DependencyCursorUrls.Any())
{
_logger.LogInformation("Depending on cursors:{DependencyCursorUrls}", _options.Value.DependencyCursorUrls);
backCursor = new AggregateCursor(_options
.Value
.DependencyCursorUrls.Select(r => new HttpReadCursor(new Uri(r), _handlerFunc)));
}
else
{
_logger.LogInformation("Depending on no cursors, meaning the job will process up to the latest catalog information.");
backCursor = MemoryCursor.CreateMax();
}
var frontCursorStorage = _storageFactory.Create();
var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri);
var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue);
// Log information about where state will be kept.
_logger.LogInformation(
"Using storage URL: {ContainerUrl}/{StoragePath}",
CloudStorageAccount.Parse(_options.Value.StorageConnectionString)
.CreateCloudBlobClient()
.GetContainerReference(_options.Value.StorageContainer)
.Uri
.AbsoluteUri,
_options.Value.NormalizeStoragePath());
_logger.LogInformation("Using cursor: {CursurUrl}", frontCursorUri.AbsoluteUri);
_logger.LogInformation("Using search service: {SearchServiceName}", _options.Value.SearchServiceName);
_logger.LogInformation("Using search index: {IndexName}", _options.Value.SearchIndexName);
_logger.LogInformation("Using hijack index: {IndexName}", _options.Value.HijackIndexName);
// Optionally create the indexes.
if (_options.Value.CreateContainersAndIndexes)
{
await _blobContainerBuilder.CreateIfNotExistsAsync();
await _indexBuilder.CreateSearchIndexIfNotExistsAsync();
await _indexBuilder.CreateHijackIndexIfNotExistsAsync();
}
await frontCursor.LoadAsync(token);
await backCursor.LoadAsync(token);
_logger.LogInformation(
"The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.",
frontCursor.Value,
backCursor.Value);
// Run the collector.
await _collector.RunAsync(
frontCursor,
backCursor,
token);
}
public string GetData() => _storageFactory.Create().Deserialize();