private async Task ExecuteCatalog2LuceneAsync(PerBatchContext context, Uri catalogIndexUri)
{
var serviceProvider = GetServiceProvider(
context,
context.LuceneContainerName,
storagePath: null);
using (var directory = serviceProvider.GetRequiredService <Lucene.Net.Store.Directory>())
using (var indexWriter = Catalog2LuceneJob.CreateIndexWriter(directory))
{
var commitEachBatch = false;
string baseAddress = null;
var collector = new SearchIndexFromCatalogCollector(
catalogIndexUri,
indexWriter,
commitEachBatch,
baseAddress,
serviceProvider.GetRequiredService <ITelemetryService>(),
serviceProvider.GetRequiredService <ILogger>(),
() => serviceProvider.GetRequiredService <HttpMessageHandler>());
await collector.RunAsync(CancellationToken.None);
}
}
protected override async Task RunInternalAsync(CancellationToken cancellationToken)
{
using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", _destination))
using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds))
using (var indexWriter = CreateIndexWriter(_directory))
{
var collector = new SearchIndexFromCatalogCollector(
index: new Uri(_source),
indexWriter: indexWriter,
commitEachBatch: false,
commitTimeout: _commitTimeout,
baseAddress: _catalogBaseAddress,
telemetryService: TelemetryService,
logger: Logger,
handlerFunc: _handlerFunc);
ReadWriteCursor front = new LuceneCursor(indexWriter, MemoryCursor.MinValue);
var back = _registration == null
? (ReadCursor)MemoryCursor.CreateMax()
: GetTheLeastAdvancedRegistrationCursor(_registration, cancellationToken);
bool run;
do
{
run = await collector.RunAsync(front, back, cancellationToken);
await collector.EnsureCommittedAsync(); // commit after each catalog page
}while (run);
}
}
public async Task ThrowsIfCommitTimesOut()
{
// Arrange
var storage = new MemoryStorage();
var storageFactory = new TestStorageFactory(name => storage.WithName(name));
MockServerHttpClientHandler mockServer;
mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage();
await mockServer.AddStorageAsync(catalogStorage);
ReadWriteCursor front = new DurableCursor(
storage.ResolveUri("cursor.json"),
storage,
MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
var commitTimeout = TimeSpan.FromSeconds(1);
var stuckDuration = TimeSpan.FromMinutes(1);
var telemetryService = new Mock <ITelemetryService>();
var indexCommitDurationMetric = new Mock <IDisposable>();
telemetryService.Setup(t => t.TrackIndexCommitDuration()).Returns(indexCommitDurationMetric.Object);
using (var testDirectory = TestDirectory.Create())
{
var luceneDirectory = new SimpleFSDirectory(new DirectoryInfo(testDirectory));
using (var indexWriter = Catalog2LuceneJob.CreateIndexWriter(luceneDirectory))
using (var stuckIndexWriter = StuckIndexWriter.FromIndexWriter(indexWriter, stuckDuration))
{
var target = new SearchIndexFromCatalogCollector(
new Uri("http://tempuri.org/index.json"),
stuckIndexWriter,
commitEachBatch: true,
commitTimeout: commitTimeout,
baseAddress: null,
telemetryService: telemetryService.Object,
logger: new TestLogger(),
handlerFunc: () => mockServer,
httpRetryStrategy: new NoRetryStrategy());
// Act & Assert
await Assert.ThrowsAsync <OperationCanceledException>(() => target.RunAsync(front, back, CancellationToken.None));
telemetryService.Verify(t => t.TrackIndexCommitDuration(), Times.Once);
telemetryService.Verify(t => t.TrackIndexCommitTimeout(), Times.Once);
indexCommitDurationMetric.Verify(m => m.Dispose(), Times.Never);
}
}
}
async Task CreateLuceneIndex(Uri catalogIndex, Func <StorageHttpMessageHandler> handlerFunc, CancellationToken cancellationToken)
{
Lucene.Net.Store.Directory luceneDirectory = new RAMDirectory();
var collector = new SearchIndexFromCatalogCollector(
catalogIndex,
luceneDirectory,
null,
handlerFunc);
await collector.Run(
new MemoryCursor(DateTime.MinValue.ToUniversalTime()),
new MemoryCursor(DateTime.MaxValue.ToUniversalTime()),
cancellationToken);
ILogger logger = new DebugLogger("Lucene");
ILoader loader = new AuxillaryIndexLoader();
SearcherManager = new NuGetSearcherManager("memory", logger, luceneDirectory, loader);
SearcherManager.RegistrationBaseAddress["http"] = new Uri(_baseAddress, Registration);
SearcherManager.RegistrationBaseAddress["https"] = new Uri(_baseAddress, Registration);
SearcherManager.Open();
}
protected override async Task RunInternal(CancellationToken cancellationToken)
{
using (var indexWriter = CreateIndexWriter(_directory))
{
var collector = new SearchIndexFromCatalogCollector(
Logger,
index: new Uri(_source),
indexWriter: indexWriter,
commitEachBatch: false,
baseAddress: _catalogBaseAddress,
handlerFunc: _handlerFunc);
ReadWriteCursor front = new LuceneCursor(indexWriter, MemoryCursor.MinValue);
var back = _registration == null
? (ReadCursor)MemoryCursor.CreateMax()
: new HttpReadCursor(new Uri(_registration), _handlerFunc);
bool run;
do
{
run = await collector.Run(front, back, cancellationToken);
collector.EnsureCommitted(); // commit after each catalog page
}
while (run);
}
}
public async Task DoesNotSkipPackagesWhenExceptionOccurs(string catalogUri, string expectedCursorBeforeRetry)
{
// Arrange
var storage = new MemoryStorage();
var storageFactory = new TestStorageFactory(name => storage.WithName(name));
MockServerHttpClientHandler mockServer;
mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit();
await mockServer.AddStorageAsync(catalogStorage);
// Make the first request for a catalog leaf node fail. This will cause the registration collector
// to fail the first time but pass the second time.
FailFirstRequest(mockServer, catalogUri);
expectedCursorBeforeRetry = expectedCursorBeforeRetry ?? MemoryCursor.MinValue.ToString("O");
ReadWriteCursor front = new DurableCursor(
storage.ResolveUri("cursor.json"),
storage,
MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
using (var testDirectory = TestDirectory.Create())
{
var luceneDirectory = new SimpleFSDirectory(new DirectoryInfo(testDirectory));
using (var indexWriter = Catalog2LuceneJob.CreateIndexWriter(luceneDirectory))
{
var target = new SearchIndexFromCatalogCollector(
new Uri("http://tempuri.org/index.json"),
indexWriter,
commitEachBatch: true,
baseAddress: null,
telemetryService: new Mock <ITelemetryService>().Object,
logger: new TestLogger(),
handlerFunc: () => mockServer);
// Act
await Assert.ThrowsAsync <Exception>(() => target.RunAsync(front, back, CancellationToken.None));
var cursorBeforeRetry = front.Value;
await target.RunAsync(front, back, CancellationToken.None);
var cursorAfterRetry = front.Value;
// Assert
var reader = indexWriter.GetReader();
var documents = Enumerable
.Range(0, reader.NumDeletedDocs + reader.NumDocs())
.Where(i => !reader.IsDeleted(i))
.Select(i => reader.Document(i))
.ToList();
Assert.Equal(4, documents.Count);
var documentsByType = documents
.ToLookup(doc => doc
.fields_ForNUnit
.FirstOrDefault(f => f.Name == "@type")?
.StringValue);
var commitDocuments = documentsByType[Schema.DataTypes.CatalogInfastructure.AbsoluteUri.ToString()].ToList();
var packageDocuments = documentsByType[null].ToList();
Assert.Equal(1, commitDocuments.Count);
Assert.Equal(3, packageDocuments.Count);
Assert.Equal(
"UnlistedPackage",
packageDocuments[0].fields_ForNUnit.FirstOrDefault(x => x.Name == Constants.LucenePropertyId)?.StringValue);
Assert.Equal(
"ListedPackage",
packageDocuments[1].fields_ForNUnit.FirstOrDefault(x => x.Name == Constants.LucenePropertyId)?.StringValue);
Assert.Equal(
"AnotherPackage",
packageDocuments[2].fields_ForNUnit.FirstOrDefault(x => x.Name == Constants.LucenePropertyId)?.StringValue);
Assert.Equal(DateTime.Parse(expectedCursorBeforeRetry).ToUniversalTime(), cursorBeforeRetry);
Assert.Equal(DateTime.Parse("2015-10-12T10:08:55.3335317Z").ToUniversalTime(), cursorAfterRetry);
}
}
}
