protected override async Task RunInternalAsync(CancellationToken cancellationToken)
{
using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", _destination))
using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds))
using (var indexWriter = CreateIndexWriter(_directory))
{
var collector = new SearchIndexFromCatalogCollector(
index: new Uri(_source),
indexWriter: indexWriter,
commitEachBatch: false,
commitTimeout: _commitTimeout,
baseAddress: _catalogBaseAddress,
telemetryService: TelemetryService,
logger: Logger,
handlerFunc: _handlerFunc);
ReadWriteCursor front = new LuceneCursor(indexWriter, MemoryCursor.MinValue);
var back = _registration == null
? (ReadCursor)MemoryCursor.CreateMax()
: GetTheLeastAdvancedRegistrationCursor(_registration, cancellationToken);
bool run;
do
{
run = await collector.RunAsync(front, back, cancellationToken);
await collector.EnsureCommittedAsync(); // commit after each catalog page
}while (run);
}
}
protected override async Task RunInternalAsync(CancellationToken cancellationToken)
{
using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", _destination.AbsoluteUri))
using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds))
{
bool run;
do
{
run = await _collector.RunAsync(_front, _back, cancellationToken);
}while (run);
}
}
protected override async Task RunInternalAsync(CancellationToken cancellationToken)
{
using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", Destination.AbsoluteUri))
using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds))
using (var client = CreateHttpClient())
{
uint packagesDeleted;
uint packagesCreated;
uint packagesEdited;
client.Timeout = Timeout;
var packageCatalogItemCreator = PackageCatalogItemCreator.Create(
client,
TelemetryService,
Logger,
PreferredPackageSourceStorage);
do
{
packagesDeleted = 0;
packagesCreated = 0;
packagesEdited = 0;
// baseline timestamps
var catalogProperties = await CatalogProperties.ReadAsync(CatalogStorage, TelemetryService, cancellationToken);
var lastCreated = catalogProperties.LastCreated ?? (StartDate ?? Constants.DateTimeMinValueUtc);
var lastEdited = catalogProperties.LastEdited ?? lastCreated;
var lastDeleted = catalogProperties.LastDeleted ?? lastCreated;
if (lastDeleted == Constants.DateTimeMinValueUtc)
{
lastDeleted = SkipCreatedPackagesProcessing ? lastEdited : lastCreated;
}
try
{
if (lastDeleted > Constants.DateTimeMinValueUtc)
{
using (TelemetryService.TrackDuration(TelemetryConstants.DeletedPackagesSeconds))
{
Logger.LogInformation("CATALOG LastDeleted: {CatalogDeletedTime}", lastDeleted.ToString("O"));
var deletedPackages = await GetDeletedPackages(AuditingStorage, lastDeleted);
packagesDeleted = (uint)deletedPackages.SelectMany(x => x.Value).Count();
Logger.LogInformation("FEED DeletedPackages: {DeletedPackagesCount}", packagesDeleted);
// We want to ensure a commit only contains each package once at most.
// Therefore we segment by package id + version.
var deletedPackagesSegments = SegmentPackageDeletes(deletedPackages);
foreach (var deletedPackagesSegment in deletedPackagesSegments)
{
lastDeleted = await Deletes2Catalog(
deletedPackagesSegment, CatalogStorage, lastCreated, lastEdited, lastDeleted, cancellationToken);
// Wait for one second to ensure the next catalog commit gets a new timestamp
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
}
if (!SkipCreatedPackagesProcessing)
{
using (TelemetryService.TrackDuration(TelemetryConstants.CreatedPackagesSeconds))
{
Logger.LogInformation("CATALOG LastCreated: {CatalogLastCreatedTime}", lastCreated.ToString("O"));
var createdPackages = await GalleryDatabaseQueryService.GetPackagesCreatedSince(lastCreated, Top);
packagesCreated = (uint)createdPackages.SelectMany(x => x.Value).Count();
Logger.LogInformation("DATABASE CreatedPackages: {CreatedPackagesCount}", packagesCreated);
lastCreated = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync(
packageCatalogItemCreator,
createdPackages,
CatalogStorage,
lastCreated,
lastEdited,
lastDeleted,
MaxDegreeOfParallelism,
createdPackages : true,
updateCreatedFromEdited : false,
cancellationToken : cancellationToken,
telemetryService : TelemetryService,
logger : Logger);
}
}
using (TelemetryService.TrackDuration(TelemetryConstants.EditedPackagesSeconds))
{
Logger.LogInformation("CATALOG LastEdited: {CatalogLastEditedTime}", lastEdited.ToString("O"));
var editedPackages = await GalleryDatabaseQueryService.GetPackagesEditedSince(lastEdited, Top);
packagesEdited = (uint)editedPackages.SelectMany(x => x.Value).Count();
Logger.LogInformation("DATABASE EditedPackages: {EditedPackagesCount}", packagesEdited);
lastEdited = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync(
packageCatalogItemCreator,
editedPackages,
CatalogStorage,
lastCreated,
lastEdited,
lastDeleted,
MaxDegreeOfParallelism,
createdPackages : false,
updateCreatedFromEdited : SkipCreatedPackagesProcessing,
cancellationToken : cancellationToken,
telemetryService : TelemetryService,
logger : Logger);
}
}
finally
{
TelemetryService.TrackMetric(TelemetryConstants.DeletedPackagesCount, packagesDeleted);
if (!SkipCreatedPackagesProcessing)
{
TelemetryService.TrackMetric(TelemetryConstants.CreatedPackagesCount, packagesCreated);
}
TelemetryService.TrackMetric(TelemetryConstants.EditedPackagesCount, packagesEdited);
}
} while (packagesDeleted > 0 || packagesCreated > 0 || packagesEdited > 0);
}
}