|
public static GetHttpMessageHandlerFactory ( bool verbose, string catalogBaseAddress = null, string storageBaseAddress = null ) : Func |
||
| verbose | bool | |
| catalogBaseAddress | string | |
| storageBaseAddress | string | |
| return | Func |
|
static async Task Loop(string source, string registration, Lucene.Net.Store.Directory directory, string catalogBaseAddress, string storageBaseAddress, bool verbose, int interval)
{
Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose, catalogBaseAddress, storageBaseAddress);
CommitCollector collector = new SearchIndexFromCatalogCollector(new Uri(source), directory, catalogBaseAddress, handlerFunc);
ReadWriteCursor front = new LuceneCursor(directory, MemoryCursor.Min.Value);
ReadCursor back = (registration == null) ? (ReadCursor)MemoryCursor.Max : new HttpReadCursor(new Uri(registration), handlerFunc);
while (true)
{
bool run = false;
do
{
run = await collector.Run(front, back);
}while (run);
Thread.Sleep(interval * 1000);
}
}
static async Task ProcessPackages(string gallery, StorageFactory storageFactory, string id, string version, bool verbose)
{
int timeout = 300;
Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);
HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler {
AllowPipelining = true
};
using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(timeout);
// if teh version is specified a single package is processed otherwise all the packages corresponding to that id are processed
Uri uri = (version == null) ? MakePackageUri(gallery, id) : MakePackageUri(gallery, id, version);
SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > packages = await GetPackages(client, uri, "Created");
Trace.TraceInformation("downloading {0} packages", packages.Select(t => t.Value.Count).Sum());
Storage storage = storageFactory.Create();
// the idea here is to leave the lastCreated and lastEdited values exactly as they were
const string LastCreated = "nuget:lastCreated";
const string LastEdited = "nuget:lastEdited";
DateTime lastCreated = await GetCatalogProperty(storage, LastCreated) ?? DateTime.MinValue.ToUniversalTime();
DateTime lastEdited = await GetCatalogProperty(storage, LastEdited) ?? DateTime.MinValue.ToUniversalTime();
DateTime d = await DownloadMetadata2Catalog(client, packages, storage, lastCreated, lastEdited);
}
}
static async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool verbose, int interval)
{
CommitCollector collector = new DnxCatalogCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
{
ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress)
};
Storage storage = storageFactory.Create();
ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value);
ReadCursor back = MemoryCursor.Max;
while (true)
{
bool run = false;
do
{
run = await collector.Run(front, back);
}while (run);
Thread.Sleep(interval * 1000);
}
}
static async Task Loop(string gallery, StorageFactory storageFactory, bool verbose, int interval, DateTime?startDate)
{
Storage storage = storageFactory.Create();
const string LastCreated = "nuget:lastCreated";
const string LastEdited = "nuget:lastEdited";
int top = 20;
int timeout = 300;
while (true)
{
Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);
HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler {
AllowPipelining = true
};
using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(timeout);
// fetch and add all newly CREATED packages - in order
DateTime lastCreated = await GetCatalogProperty(storage, LastCreated) ?? (startDate ?? DateTime.MinValue.ToUniversalTime());
DateTime lastEdited = await GetCatalogProperty(storage, LastEdited) ?? lastCreated;
SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > createdPackages;
DateTime previousLastCreated = DateTime.MinValue;
do
{
Trace.TraceInformation("CATALOG LastCreated: {0}", lastCreated.ToString("O"));
createdPackages = await GetCreatedPackages(client, gallery, lastCreated, top);
Trace.TraceInformation("FEED CreatedPackages: {0}", createdPackages.Count);
lastCreated = await DownloadMetadata2Catalog(client, createdPackages, storage, lastCreated, lastEdited, createdPackages : true);
if (previousLastCreated == lastCreated)
{
break;
}
previousLastCreated = lastCreated;
}while (createdPackages.Count > 0);
// THEN fetch and add all EDITED packages - in order
SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > editedPackages;
DateTime previousLastEdited = DateTime.MinValue;
do
{
Trace.TraceInformation("CATALOG LastEdited: {0}", lastEdited.ToString("O"));
editedPackages = await GetEditedPackages(client, gallery, lastEdited, top);
Trace.TraceInformation("FEED EditedPackages: {0}", editedPackages.Count);
lastEdited = await DownloadMetadata2Catalog(client, editedPackages, storage, lastCreated, lastEdited, createdPackages : false);
if (previousLastEdited == lastEdited)
{
break;
}
previousLastEdited = lastEdited;
}while (editedPackages.Count > 0);
}
Thread.Sleep(interval * 1000);
}
}
