public static string GetPackageIdKey(CatalogCommitItem item)
{
if (item == null)
{
throw new ArgumentNullException(nameof(item));
}
return(item.PackageIdentity.Id.ToLowerInvariant());
}
public static CatalogIndexEntry Create(CatalogCommitItem commitItem)
{
if (commitItem == null)
{
throw new ArgumentNullException(nameof(commitItem));
}
return(new CatalogIndexEntry(
commitItem.Uri,
commitItem.Types,
commitItem.CommitId,
commitItem.CommitTimeStamp,
commitItem.PackageIdentity));
}
private static async Task <CatalogCommitItemBatch[]> CreateBatchesForAllAvailableItemsInPageAsync(
ReadWriteCursor front,
ReadCursor back,
JObject page,
JObject context,
CreateCommitItemBatchesAsync createCommitItemBatchesAsync)
{
IEnumerable <CatalogCommitItem> commitItems = page["items"]
.Select(item => CatalogCommitItem.Create(context, (JObject)item))
.Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value);
IEnumerable <CatalogCommitItemBatch> batches = await createCommitItemBatchesAsync(commitItems);
return(batches
.OrderBy(batch => batch.CommitTimeStamp)
.ToArray());
}
protected override FeedPackageIdentity GetKey(CatalogCommitItem item)
{
return(new FeedPackageIdentity(item.PackageIdentity));
}
protected override string GetKey(CatalogCommitItem item)
{
return(item.PackageIdentity.Id);
}
protected override async Task <bool> FetchAsync(
CollectorHttpClient client,
ReadWriteCursor front,
ReadCursor back,
CancellationToken cancellationToken)
{
IEnumerable <CatalogCommit> commits = await FetchCatalogCommitsAsync(client, front, cancellationToken);
bool acceptNextBatch = false;
foreach (CatalogCommit commit in commits)
{
JObject page = await client.GetJObjectAsync(commit.Uri, cancellationToken);
JToken context = null;
page.TryGetValue("@context", out context);
var batches = await CreateBatchesAsync(page["items"]
.Select(item => CatalogCommitItem.Create((JObject)context, (JObject)item))
.Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value));
var orderedBatches = batches
.OrderBy(batch => batch.CommitTimeStamp)
.ToList();
var lastBatch = orderedBatches.LastOrDefault();
DateTime?previousCommitTimeStamp = null;
foreach (var batch in orderedBatches)
{
// If the commit timestamp has changed from the previous batch, commit. This is important because if
// two batches have the same commit timestamp but processing the second fails, we should not
// progress the cursor forward.
if (previousCommitTimeStamp.HasValue && previousCommitTimeStamp != batch.CommitTimeStamp)
{
front.Value = previousCommitTimeStamp.Value;
await front.SaveAsync(cancellationToken);
Trace.TraceInformation("CommitCatalog.Fetch front.Value saved since timestamp changed from previous: {0}", front);
}
using (_telemetryService.TrackDuration(TelemetryConstants.ProcessBatchSeconds, new Dictionary <string, string>()
{
{ TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() }
}))
{
acceptNextBatch = await OnProcessBatchAsync(
client,
batch.Items,
context,
batch.CommitTimeStamp,
batch.CommitTimeStamp == lastBatch.CommitTimeStamp,
cancellationToken);
}
// If this is the last batch, commit the cursor.
if (ReferenceEquals(batch, lastBatch))
{
front.Value = batch.CommitTimeStamp;
await front.SaveAsync(cancellationToken);
Trace.TraceInformation("CommitCatalog.Fetch front.Value saved due to last batch: {0}", front);
}
previousCommitTimeStamp = batch.CommitTimeStamp;
Trace.TraceInformation("CommitCatalog.Fetch front.Value is: {0}", front);
if (!acceptNextBatch)
{
break;
}
}
if (!acceptNextBatch)
{
break;
}
}
return(acceptNextBatch);
}
protected abstract T GetKey(CatalogCommitItem item);
