public async Task ExecuteAsync(CancellationToken token)
{
var registrationCursors = CursorUtility.GetRegistrationCursors(_handlerFunc, _options);
var backCursor = new AggregateCursor(registrationCursors.Values);
var frontCursorPair = CursorUtility.GetComparerCursor(_storageFactory);
_logger.LogInformation("Using cursor: {CursurUrl}", frontCursorPair.Key);
var frontCursor = frontCursorPair.Value;
var container = _storageAccount
.CreateCloudBlobClient()
.GetContainerReference(_options.Value.StorageContainer);
await container.CreateIfNotExistsAsync();
await container.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob });
await frontCursor.LoadAsync(token);
await backCursor.LoadAsync(token);
_logger.LogInformation(
"The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.",
frontCursor.Value,
backCursor.Value);
// Run the collector.
await _collector.RunAsync(
frontCursor,
backCursor,
token);
}
public SearchEndpoint(
string instanceName,
IReadOnlyList <Uri> cursorUris,
Uri baseUri,
Func <HttpMessageHandler> messageHandlerFactory)
{
Cursor = new AggregateCursor(cursorUris.Select(c => new HttpReadCursor(c, messageHandlerFactory)));
InstanceName = instanceName;
BaseUri = baseUri;
}
private async Task ExecuteAsync(CancellationToken token)
{
// Initialize the cursors.
ReadCursor backCursor;
if (_options.Value.DependencyCursorUrls != null &&
_options.Value.DependencyCursorUrls.Any())
{
_logger.LogInformation("Depending on cursors: {DependencyCursorUrls}", _options.Value.DependencyCursorUrls);
backCursor = new AggregateCursor(_options
.Value
.DependencyCursorUrls.Select(r => new HttpReadCursor(new Uri(r), _handlerFunc)));
}
else
{
_logger.LogInformation("Depending on no cursors, meaning the job will process up to the latest catalog information.");
backCursor = MemoryCursor.CreateMax();
}
var frontCursorStorage = _storageFactory.Create();
var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri);
var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue);
_logger.LogInformation("Using cursor: {CursurUrl}", frontCursorUri.AbsoluteUri);
LogContainerUrl(HiveType.Legacy, c => c.LegacyStorageContainer);
LogContainerUrl(HiveType.Gzipped, c => c.GzippedStorageContainer);
LogContainerUrl(HiveType.SemVer2, c => c.SemVer2StorageContainer);
// Optionally create the containers.
if (_options.Value.CreateContainers)
{
await CreateContainerIfNotExistsAsync(c => c.LegacyStorageContainer);
await CreateContainerIfNotExistsAsync(c => c.GzippedStorageContainer);
await CreateContainerIfNotExistsAsync(c => c.SemVer2StorageContainer);
}
await frontCursor.LoadAsync(token);
await backCursor.LoadAsync(token);
_logger.LogInformation(
"The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.",
frontCursor.Value,
backCursor.Value);
// Run the collector.
await _collector.RunAsync(
frontCursor,
backCursor,
token);
}
public async Task ReturnsLeastValue(IEnumerable <DateTime> dates)
{
// Arrange
var cursors = dates.Select(d => CreateReadCursor(d));
var aggregateCursor = new AggregateCursor(cursors);
// Act
await aggregateCursor.LoadAsync(CancellationToken.None);
var value = aggregateCursor.Value;
// Assert
Assert.Equal(dates.Min(), value);
}
/// <summary>
/// Constructs a <see cref="ValidationCollector"/> from inputs and returns a <see cref="Result"/>.
/// </summary>
/// <param name="queue">Queue that the <see cref="ValidationCollector"/> queues packages to.</param>
/// <param name="catalogIndexUrl">Url of the catalog that the <see cref="ValidationCollector"/> should run on.</param>
/// <param name="monitoringStorageFactory">Storage where the cursors used by the <see cref="ValidationCollector"/> are stored.</param>
/// <param name="endpointInputs">Endpoints that validations will be run on for queued packages.</param>
/// <param name="messageHandlerFactory">Used by <see cref="ValidationCollector"/> to construct a <see cref="CollectorHttpClient"/>.</param>
public Result Create(
IStorageQueue <PackageValidatorContext> queue,
string catalogIndexUrl,
Persistence.IStorageFactory monitoringStorageFactory,
IEnumerable <EndpointFactory.Input> endpointInputs,
ITelemetryService telemetryService,
Func <HttpMessageHandler> messageHandlerFactory)
{
var collector = new ValidationCollector(
queue,
new Uri(catalogIndexUrl),
telemetryService,
_loggerFactory.CreateLogger <ValidationCollector>(),
messageHandlerFactory);
var front = GetFront(monitoringStorageFactory);
var back = new AggregateCursor(endpointInputs.Select(input => new HttpReadCursor(input.CursorUri, messageHandlerFactory)));
return(new Result(collector, front, back));
}
private async Task ExecuteAsync(CancellationToken token)
{
// Initialize the cursors.
ReadCursor backCursor;
if (_options.Value.DependencyCursorUrls != null &&
_options.Value.DependencyCursorUrls.Any())
{
_logger.LogInformation("Depending on cursors:{DependencyCursorUrls}", _options.Value.DependencyCursorUrls);
backCursor = new AggregateCursor(_options
.Value
.DependencyCursorUrls.Select(r => new HttpReadCursor(new Uri(r), _handlerFunc)));
}
else
{
_logger.LogInformation("Depending on no cursors, meaning the job will process up to the latest catalog information.");
backCursor = MemoryCursor.CreateMax();
}
var frontCursorStorage = _storageFactory.Create();
var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri);
var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue);
// Log information about where state will be kept.
_logger.LogInformation(
"Using storage URL: {ContainerUrl}/{StoragePath}",
CloudStorageAccount.Parse(_options.Value.StorageConnectionString)
.CreateCloudBlobClient()
.GetContainerReference(_options.Value.StorageContainer)
.Uri
.AbsoluteUri,
_options.Value.NormalizeStoragePath());
_logger.LogInformation("Using cursor: {CursurUrl}", frontCursorUri.AbsoluteUri);
_logger.LogInformation("Using search service: {SearchServiceName}", _options.Value.SearchServiceName);
_logger.LogInformation("Using search index: {IndexName}", _options.Value.SearchIndexName);
_logger.LogInformation("Using hijack index: {IndexName}", _options.Value.HijackIndexName);
// Optionally create the indexes.
if (_options.Value.CreateContainersAndIndexes)
{
await _blobContainerBuilder.CreateIfNotExistsAsync();
await _indexBuilder.CreateSearchIndexIfNotExistsAsync();
await _indexBuilder.CreateHijackIndexIfNotExistsAsync();
}
await frontCursor.LoadAsync(token);
await backCursor.LoadAsync(token);
_logger.LogInformation(
"The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.",
frontCursor.Value,
backCursor.Value);
// Run the collector.
await _collector.RunAsync(
frontCursor,
backCursor,
token);
}
