public static string GetPackageIdKey(CatalogCommitItem item) { if (item == null) { throw new ArgumentNullException(nameof(item)); } return(item.PackageIdentity.Id.ToLowerInvariant()); }
public static CatalogIndexEntry Create(CatalogCommitItem commitItem) { if (commitItem == null) { throw new ArgumentNullException(nameof(commitItem)); } return(new CatalogIndexEntry( commitItem.Uri, commitItem.Types, commitItem.CommitId, commitItem.CommitTimeStamp, commitItem.PackageIdentity)); }
private static async Task <CatalogCommitItemBatch[]> CreateBatchesForAllAvailableItemsInPageAsync( ReadWriteCursor front, ReadCursor back, JObject page, JObject context, CreateCommitItemBatchesAsync createCommitItemBatchesAsync) { IEnumerable <CatalogCommitItem> commitItems = page["items"] .Select(item => CatalogCommitItem.Create(context, (JObject)item)) .Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value); IEnumerable <CatalogCommitItemBatch> batches = await createCommitItemBatchesAsync(commitItems); return(batches .OrderBy(batch => batch.CommitTimeStamp) .ToArray()); }
protected override FeedPackageIdentity GetKey(CatalogCommitItem item) { return(new FeedPackageIdentity(item.PackageIdentity)); }
protected override string GetKey(CatalogCommitItem item) { return(item.PackageIdentity.Id); }
protected override async Task <bool> FetchAsync( CollectorHttpClient client, ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken) { IEnumerable <CatalogCommit> commits = await FetchCatalogCommitsAsync(client, front, cancellationToken); bool acceptNextBatch = false; foreach (CatalogCommit commit in commits) { JObject page = await client.GetJObjectAsync(commit.Uri, cancellationToken); JToken context = null; page.TryGetValue("@context", out context); var batches = await CreateBatchesAsync(page["items"] .Select(item => CatalogCommitItem.Create((JObject)context, (JObject)item)) .Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value)); var orderedBatches = batches .OrderBy(batch => batch.CommitTimeStamp) .ToList(); var lastBatch = orderedBatches.LastOrDefault(); DateTime?previousCommitTimeStamp = null; foreach (var batch in orderedBatches) { // If the commit timestamp has changed from the previous batch, commit. This is important because if // two batches have the same commit timestamp but processing the second fails, we should not // progress the cursor forward. if (previousCommitTimeStamp.HasValue && previousCommitTimeStamp != batch.CommitTimeStamp) { front.Value = previousCommitTimeStamp.Value; await front.SaveAsync(cancellationToken); Trace.TraceInformation("CommitCatalog.Fetch front.Value saved since timestamp changed from previous: {0}", front); } using (_telemetryService.TrackDuration(TelemetryConstants.ProcessBatchSeconds, new Dictionary <string, string>() { { TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() } })) { acceptNextBatch = await OnProcessBatchAsync( client, batch.Items, context, batch.CommitTimeStamp, batch.CommitTimeStamp == lastBatch.CommitTimeStamp, cancellationToken); } // If this is the last batch, commit the cursor. if (ReferenceEquals(batch, lastBatch)) { front.Value = batch.CommitTimeStamp; await front.SaveAsync(cancellationToken); Trace.TraceInformation("CommitCatalog.Fetch front.Value saved due to last batch: {0}", front); } previousCommitTimeStamp = batch.CommitTimeStamp; Trace.TraceInformation("CommitCatalog.Fetch front.Value is: {0}", front); if (!acceptNextBatch) { break; } } if (!acceptNextBatch) { break; } } return(acceptNextBatch); }
protected abstract T GetKey(CatalogCommitItem item);