protected override Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context)
{
ChecksumCollectorEventSource.Log.ProcessingBatch(BatchCount, items.Count);
foreach (var item in items)
{
string type = item.Value<string>("@type");
string url = item.Value<string>("url");
ChecksumCollectorEventSource.Log.CollectingItem(type, url);
var key = Int32.Parse(item.Value<string>("galleryKey"));
if (String.Equals(type, "nuget:Package", StringComparison.Ordinal))
{
var checksum = item.Value<string>("galleryChecksum");
var id = item.Value<string>("nuget:id");
var version = item.Value<string>("nuget:version");
Checksums.Data[key] = new JObject(
new JProperty("checksum", checksum),
new JProperty("id", id),
new JProperty("version", version));
}
else if (String.Equals(type, "nuget:PackageDeletion", StringComparison.Ordinal))
{
Checksums.Data.Remove(key);
}
ChecksumCollectorEventSource.Log.CollectedItem();
}
ChecksumCollectorEventSource.Log.ProcessedBatch();
return Task.FromResult(0);
}
protected override async Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context)
{
List<Task<IGraph>> tasks = new List<Task<IGraph>>();
foreach (JObject item in items)
{
if (Utils.IsType(context, item, Constants.Package))
{
Uri itemUri = item["url"].ToObject<Uri>();
tasks.Add(client.GetGraphAsync(itemUri));
}
}
if (tasks.Count > 0)
{
await Task.WhenAll(tasks.ToArray());
TripleStore store = new TripleStore();
foreach (Task<IGraph> task in tasks)
{
store.Add(task.Result, true);
}
await ProcessStore(store);
}
}
public async Task Run(Uri index, DateTime last)
{
using (CollectorHttpClient client = new CollectorHttpClient())
{
await Fetch(client, index, last);
RequestCount = client.RequestCount;
}
}
public CatalogUpdater(CatalogWriter writer, ChecksumRecords checksums, CollectorHttpClient client)
{
DatabaseChecksumBatchSize = DefaultDatabaseChecksumBatchSize;
CatalogAddBatchSize = DefaultCatalogAddBatchSize;
_writer = writer;
_checksums = checksums;
_client = client;
}
protected override async Task<CollectorCursor> Fetch(CollectorHttpClient client, Uri index, CollectorCursor last)
{
ChecksumCollectorEventSource.Log.Collecting(index.ToString(), ((DateTime)last).ToString("O"));
// Run the collector
var cursor = await base.Fetch(client, index, last);
// Update the cursor
Checksums.Cursor = cursor;
ChecksumCollectorEventSource.Log.Collected();
return cursor;
}
protected override async Task Fetch(CollectorHttpClient client, Uri index, DateTime last)
{
IList<JObject> items = new List<JObject>();
JObject root = await client.GetJObjectAsync(index);
JToken context = null;
root.TryGetValue("@context", out context);
IEnumerable<JToken> rootItems = root["item"].OrderBy(item => item["timeStamp"]["@value"].ToObject<DateTime>());
foreach (JObject rootItem in rootItems)
{
DateTime pageTimeStamp = rootItem["timeStamp"]["@value"].ToObject<DateTime>();
if (pageTimeStamp > last)
{
Uri pageUri = rootItem["url"].ToObject<Uri>();
JObject page = await client.GetJObjectAsync(pageUri);
IEnumerable<JToken> pageItems = page["item"].OrderBy(item => item["timeStamp"]["@value"].ToObject<DateTime>());
foreach (JObject pageItem in pageItems)
{
DateTime itemTimeStamp = pageItem["timeStamp"]["@value"].ToObject<DateTime>();
if (itemTimeStamp > last)
{
Uri itemUri = pageItem["url"].ToObject<Uri>();
items.Add(pageItem);
if (items.Count == _batchSize)
{
await ProcessBatch(client, items, (JObject)context);
BatchCount++;
items.Clear();
}
}
}
}
}
if (items.Count > 0)
{
await ProcessBatch(client, items, (JObject)context);
BatchCount++;
}
}
protected override async Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context)
{
List<Task<JObject>> tasks = new List<Task<JObject>>();
foreach (JObject item in items)
{
Uri itemUri = new Uri(item["url"].ToString());
string type = item["@type"].ToString();
if (type == "TestItem")
{
tasks.Add(client.GetJObjectAsync(itemUri));
}
}
await Task.WhenAll(tasks.ToArray());
foreach (Task<JObject> task in tasks)
{
Console.WriteLine(task.Result["name"]);
}
}
protected async override Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context)
{
List<Task<string>> tasks = new List<Task<string>>();
foreach (JObject item in items)
{
DateTime itemMinDownloadTimestamp = item["http://nuget.org/schema#minDownloadTimestamp"]["@value"].ToObject<DateTime>();
DateTime itemMaxDownloadTimestamp = item["http://nuget.org/schema#maxDownloadTimestamp"]["@value"].ToObject<DateTime>();
if (SelectItem(itemMinDownloadTimestamp, itemMaxDownloadTimestamp))
{
Uri itemUri = item["url"].ToObject<Uri>();
tasks.Add(client.GetStringAsync(itemUri));
}
}
if (tasks.Count > 0)
{
await Task.WhenAll(tasks.ToArray());
foreach (Task<string> task in tasks)
{
JArray statisticsPage = JArray.Parse(task.Result);
foreach (JArray row in statisticsPage)
{
DateTime rowTimeStamp = row[1].ToObject<DateTime>();
if (SelectRow(rowTimeStamp))
{
Count++;
}
}
}
}
}
protected abstract Task Fetch(CollectorHttpClient client, Uri index, DateTime last);
protected abstract Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context);
protected abstract Task<CollectorCursor> Fetch(CollectorHttpClient client, Uri index, CollectorCursor last);
public async Task<CollectorCursor> Run(CollectorHttpClient client, Uri index, CollectorCursor last)
{
CollectorCursor cursor = await Fetch(client, index, last);
RequestCount = client.RequestCount;
return cursor;
}
