protected override async Task<CollectorCursor> Fetch(CollectorHttpClient client, Uri index, CollectorCursor last)
{
CollectorCursor cursor = last;
DateTime lastDateTime = (DateTime)cursor;
JObject root = await client.GetJObjectAsync(index);
List<Task<JObject>> tasks = new List<Task<JObject>>();
foreach (JObject rootItem in root["items"])
{
DateTime pageTimeStamp = rootItem["commitTimestamp"].ToObject<DateTime>();
if (pageTimeStamp > lastDateTime)
{
cursor = pageTimeStamp;
int count = int.Parse(rootItem["count"].ToString());
Total += count;
}
}
return cursor;
}
public async Task<CollectorCursor> Run(Uri index, CollectorCursor last, HttpMessageHandler handler = null)
{
CollectorCursor cursor;
using (CollectorHttpClient client = handler == null ? new CollectorHttpClient() : new CollectorHttpClient(handler))
{
cursor = await Fetch(client, index, last);
RequestCount = client.RequestCount;
}
return cursor;
}
protected override async Task<CollectorCursor> Fetch(CollectorHttpClient client, Uri index, CollectorCursor last)
{
ChecksumCollectorEventSource.Log.Collecting(index.ToString(), ((DateTime)last).ToString("O"));
// Run the collector
var cursor = await base.Fetch(client, index, last);
// Update the cursor
Checksums.Cursor = cursor;
ChecksumCollectorEventSource.Log.Collected();
return cursor;
}
public void Push(PushArgs args)
{
DirectoryInfo destinationDir = new DirectoryInfo(args.Destination);
if (!destinationDir.Exists)
{
Console.WriteLine("The destination folder does not exist.");
Environment.Exit(1);
}
FileInfo nupkg = new FileInfo(args.Nupkg);
if (!nupkg.Exists)
{
Console.WriteLine("The given nupkg does not exist.");
Environment.Exit(1);
}
Config config = new Config("http://localhost:8000/", destinationDir.FullName);
Queue <BuildStep> steps = new Queue <BuildStep>();
Queue <string> nupkgs = new Queue <string>();
nupkgs.Enqueue(nupkg.FullName);
// start the cursor before any operators take place
CollectorCursor cursor = new CollectorCursor(DateTime.UtcNow);
steps.Enqueue(new CopyPackagesStep(config, nupkgs));
steps.Enqueue(new CatalogStep(config, nupkgs));
steps.Enqueue(new ResolverStep(config, cursor));
while (steps.Count > 0)
{
BuildStep step = steps.Dequeue();
RunStep(step);
step.Dispose();
}
}
public void Push(PushArgs args)
{
DirectoryInfo destinationDir = new DirectoryInfo(args.Destination);
if (!destinationDir.Exists)
{
Console.WriteLine("The destination folder does not exist.");
Environment.Exit(1);
}
FileInfo nupkg = new FileInfo(args.Nupkg);
if (!nupkg.Exists)
{
Console.WriteLine("The given nupkg does not exist.");
Environment.Exit(1);
}
Config config = new Config("http://localhost:8000/", destinationDir.FullName);
Queue<BuildStep> steps = new Queue<BuildStep>();
Queue<string> nupkgs = new Queue<string>();
nupkgs.Enqueue(nupkg.FullName);
// start the cursor before any operators take place
CollectorCursor cursor = new CollectorCursor(DateTime.UtcNow);
steps.Enqueue(new CopyPackagesStep(config, nupkgs));
steps.Enqueue(new CatalogStep(config, nupkgs));
steps.Enqueue(new ResolverStep(config, cursor));
while (steps.Count > 0)
{
BuildStep step = steps.Dequeue();
RunStep(step);
step.Dispose();
}
}
static async Task BuildSearchIndexFromCatalog(string catalogPath, string resolverBlobPath, string dest, bool resetIndex)
{
Directory dir = new Lucene.Net.Store.SimpleFSDirectory(new System.IO.DirectoryInfo(dest));
if (resetIndex)
{
PackageIndexing.CreateNewEmptyIndex(dir);
}
var catalog = new SearchIndexFromCatalogCollector(dir, "{0}/{1}.json");
if (resolverBlobPath != null)
{
catalog.DependentCollections = new List <Uri> {
new Uri(resolverBlobPath)
};
}
CollectorCursor cursor;
//try
//{
// IndexWriter writer = SearchIndexFromCatalogCollector.CreateIndexWriter(dir, false);
//}
//catch
//{
cursor = CollectorCursor.None;
//}
CollectorCursor finalCursor = await catalog.Run(
new NuGet.Services.Metadata.Catalog.Collecting.CollectorHttpClient(),
new Uri(catalogPath),
cursor);
return;
}
public ResolverStep(Config config, CollectorCursor cursor)
: base(config, "PackageRegistrations")
{
_cursor = cursor;
}
public ResolverStep(Config config, CollectorCursor cursor)
: base(config, "PackageRegistrations")
{
_cursor = cursor;
}
private async Task StoreCursor(NuGet.Services.Metadata.Catalog.Persistence.Storage storage, Uri cursorUri, CollectorCursor value)
{
if (!Equals(value, CollectorCursor.None))
{
Log.StoringCursor(value.Value);
var cursorContent = new JObject {
{ "http://schema.nuget.org/collectors/resolver#cursor", new JObject {
{ "@value", value.Value },
{ "@type", "http://www.w3.org/2001/XMLSchema#dateTime" }
} },
{ "http://schema.nuget.org/collectors/resolver#source", CatalogIndexUrl }
}.ToString();
await storage.Save(cursorUri, new StringStorageContent(
cursorContent,
contentType : "application/json",
cacheControl : "no-store"));
Log.StoredCursor();
}
}
protected abstract Task<CollectorCursor> Fetch(CollectorHttpClient client, Uri index, CollectorCursor last);
public async Task<CollectorCursor> Run(CollectorHttpClient client, Uri index, CollectorCursor last)
{
CollectorCursor cursor = await Fetch(client, index, last);
RequestCount = client.RequestCount;
return cursor;
}
