protected override Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context) { ChecksumCollectorEventSource.Log.ProcessingBatch(BatchCount, items.Count); foreach (var item in items) { string type = item.Value<string>("@type"); string url = item.Value<string>("url"); ChecksumCollectorEventSource.Log.CollectingItem(type, url); var key = Int32.Parse(item.Value<string>("galleryKey")); if (String.Equals(type, "nuget:Package", StringComparison.Ordinal)) { var checksum = item.Value<string>("galleryChecksum"); var id = item.Value<string>("nuget:id"); var version = item.Value<string>("nuget:version"); Checksums.Data[key] = new JObject( new JProperty("checksum", checksum), new JProperty("id", id), new JProperty("version", version)); } else if (String.Equals(type, "nuget:PackageDeletion", StringComparison.Ordinal)) { Checksums.Data.Remove(key); } ChecksumCollectorEventSource.Log.CollectedItem(); } ChecksumCollectorEventSource.Log.ProcessedBatch(); return Task.FromResult(0); }
protected override async Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context) { List<Task<IGraph>> tasks = new List<Task<IGraph>>(); foreach (JObject item in items) { if (Utils.IsType(context, item, Constants.Package)) { Uri itemUri = item["url"].ToObject<Uri>(); tasks.Add(client.GetGraphAsync(itemUri)); } } if (tasks.Count > 0) { await Task.WhenAll(tasks.ToArray()); TripleStore store = new TripleStore(); foreach (Task<IGraph> task in tasks) { store.Add(task.Result, true); } await ProcessStore(store); } }
public async Task Run(Uri index, DateTime last) { using (CollectorHttpClient client = new CollectorHttpClient()) { await Fetch(client, index, last); RequestCount = client.RequestCount; } }
public CatalogUpdater(CatalogWriter writer, ChecksumRecords checksums, CollectorHttpClient client) { DatabaseChecksumBatchSize = DefaultDatabaseChecksumBatchSize; CatalogAddBatchSize = DefaultCatalogAddBatchSize; _writer = writer; _checksums = checksums; _client = client; }
protected override async Task<CollectorCursor> Fetch(CollectorHttpClient client, Uri index, CollectorCursor last) { ChecksumCollectorEventSource.Log.Collecting(index.ToString(), ((DateTime)last).ToString("O")); // Run the collector var cursor = await base.Fetch(client, index, last); // Update the cursor Checksums.Cursor = cursor; ChecksumCollectorEventSource.Log.Collected(); return cursor; }
protected override async Task Fetch(CollectorHttpClient client, Uri index, DateTime last) { IList<JObject> items = new List<JObject>(); JObject root = await client.GetJObjectAsync(index); JToken context = null; root.TryGetValue("@context", out context); IEnumerable<JToken> rootItems = root["item"].OrderBy(item => item["timeStamp"]["@value"].ToObject<DateTime>()); foreach (JObject rootItem in rootItems) { DateTime pageTimeStamp = rootItem["timeStamp"]["@value"].ToObject<DateTime>(); if (pageTimeStamp > last) { Uri pageUri = rootItem["url"].ToObject<Uri>(); JObject page = await client.GetJObjectAsync(pageUri); IEnumerable<JToken> pageItems = page["item"].OrderBy(item => item["timeStamp"]["@value"].ToObject<DateTime>()); foreach (JObject pageItem in pageItems) { DateTime itemTimeStamp = pageItem["timeStamp"]["@value"].ToObject<DateTime>(); if (itemTimeStamp > last) { Uri itemUri = pageItem["url"].ToObject<Uri>(); items.Add(pageItem); if (items.Count == _batchSize) { await ProcessBatch(client, items, (JObject)context); BatchCount++; items.Clear(); } } } } } if (items.Count > 0) { await ProcessBatch(client, items, (JObject)context); BatchCount++; } }
protected override async Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context) { List<Task<JObject>> tasks = new List<Task<JObject>>(); foreach (JObject item in items) { Uri itemUri = new Uri(item["url"].ToString()); string type = item["@type"].ToString(); if (type == "TestItem") { tasks.Add(client.GetJObjectAsync(itemUri)); } } await Task.WhenAll(tasks.ToArray()); foreach (Task<JObject> task in tasks) { Console.WriteLine(task.Result["name"]); } }
protected async override Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context) { List<Task<string>> tasks = new List<Task<string>>(); foreach (JObject item in items) { DateTime itemMinDownloadTimestamp = item["http://nuget.org/schema#minDownloadTimestamp"]["@value"].ToObject<DateTime>(); DateTime itemMaxDownloadTimestamp = item["http://nuget.org/schema#maxDownloadTimestamp"]["@value"].ToObject<DateTime>(); if (SelectItem(itemMinDownloadTimestamp, itemMaxDownloadTimestamp)) { Uri itemUri = item["url"].ToObject<Uri>(); tasks.Add(client.GetStringAsync(itemUri)); } } if (tasks.Count > 0) { await Task.WhenAll(tasks.ToArray()); foreach (Task<string> task in tasks) { JArray statisticsPage = JArray.Parse(task.Result); foreach (JArray row in statisticsPage) { DateTime rowTimeStamp = row[1].ToObject<DateTime>(); if (SelectRow(rowTimeStamp)) { Count++; } } } } }
protected abstract Task Fetch(CollectorHttpClient client, Uri index, DateTime last);
protected abstract Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context);
protected abstract Task<CollectorCursor> Fetch(CollectorHttpClient client, Uri index, CollectorCursor last);
public async Task<CollectorCursor> Run(CollectorHttpClient client, Uri index, CollectorCursor last) { CollectorCursor cursor = await Fetch(client, index, last); RequestCount = client.RequestCount; return cursor; }