protected override async Task <bool> ProcessBatch(CollectorHttpClient client, IList <Newtonsoft.Json.Linq.JObject> items, Newtonsoft.Json.Linq.JObject context)
{
_cache = new NuGet3.Client.Core.JsonLdPageCache(client);
PerfEventTracker perfTracker = new PerfEventTracker();
TextWriter log = Console.Out;
Task <Document>[] packages = items.Select(x => MakePackage(client, x)).Where(x => x != null).ToArray();
await Task.WhenAll(packages);
foreach (Task <Document> pkg in packages)
{
if (pkg != null && pkg.Result != null)
{
Console.WriteLine("Package: {0}", pkg.Result.Get("Id"));
}
}
using (IndexWriter indexWriter = CreateIndexWriter(_directory, create: false))
{
foreach (Document doc in packages.Select(x => x.Result))
{
if (doc != null)
{
Console.WriteLine("Index document: {0}", doc.Get("Id"));
indexWriter.AddDocument(doc);
}
}
indexWriter.Commit();
}
return(true);
}
// this function will incrementally build an index from the gallery using a high water mark stored in the commit metadata
// this function is useful for building a fresh index as in that case it is more efficient than diff-ing approach
public static void RebuildIndex(string sqlConnectionString, Lucene.Net.Store.Directory directory,TextWriter log = null, PerfEventTracker perfTracker = null)
{
perfTracker = perfTracker ?? new PerfEventTracker();
log = log ?? DefaultTraceWriter;
Stopwatch sw = new Stopwatch();
sw.Start();
using (perfTracker.TrackEvent("RebuildIndex", String.Empty))
{
// Empty the index, we're rebuilding
CreateNewEmptyIndex(directory);
log.WriteLine("get curated feeds by PackageRegistration");
IDictionary<int, IEnumerable<string>> feeds = GalleryExport.GetFeedsByPackageRegistration(sqlConnectionString, log, verbose: false);
int highestPackageKey = 0;
while (true)
{
log.WriteLine("get the checksums from the gallery");
IDictionary<int, int> checksums = GalleryExport.FetchGalleryChecksums(sqlConnectionString, highestPackageKey);
log.WriteLine("get packages from gallery where the Package.Key > {0}", highestPackageKey);
List<Package> packages = GalleryExport.GetPublishedPackagesSince(sqlConnectionString, highestPackageKey, log, verbose: false);
if (packages.Count == 0)
{
break;
}
log.WriteLine("associate the feeds and checksum data with each packages");
List<IndexDocumentData> indexDocumentData = MakeIndexDocumentData(packages, feeds, checksums);
highestPackageKey = indexDocumentData.Max(d => d.Package.Key);
AddPackagesToIndex(indexDocumentData, directory, log, perfTracker);
// Summarize performance
// (Save some time by not bothering if the log is "null")
if (!ReferenceEquals(TextWriter.Null, log) && !ReferenceEquals(PerfEventTracker.Null, perfTracker))
{
SummarizePerf(log, perfTracker);
}
}
}
SummarizePerf(log, perfTracker);
sw.Stop();
log.WriteLine("all done, took {0}", sw.Elapsed);
}
private static void SummarizePerf(TextWriter log, PerfEventTracker perfTracker)
{
log.WriteLine("Perf Summary:");
foreach (var evt in perfTracker.GetEvents())
{
var summary = perfTracker.GetSummary(evt);
log.WriteLine(" {0} Avg:{5:0.00}ms Max: {1:0.00}ms({2}) Min: {3:0.00}ms({4})",
evt,
summary.Max.Duration.TotalMilliseconds,
summary.Max.Payload,
summary.Min.Duration.TotalMilliseconds,
summary.Min.Payload,
summary.Average.TotalMilliseconds);
}
perfTracker.Clear();
}
private static void AddToIndex(Lucene.Net.Store.Directory directory, List<IndexDocumentData> rangeToIndex, TextWriter log, PerfEventTracker perfTracker)
{
log.WriteLine("begin AddToIndex");
int highestPackageKey = -1;
using (IndexWriter indexWriter = CreateIndexWriter(directory, create: false))
{
// Just write the document to index. No Facet.
foreach (IndexDocumentData data in rangeToIndex)
{
indexWriter.AddDocument(CreateLuceneDocument(data));
}
highestPackageKey = rangeToIndex.Max(i => i.Package.Key);
log.WriteLine("about to commit {0} packages", rangeToIndex.Count);
IDictionary<string, string> commitUserData = indexWriter.GetReader().CommitUserData;
string lastEditsIndexTime = commitUserData["last-edits-index-time"];
if (lastEditsIndexTime == null)
{
// this should never happen but if it did Lucene would throw
lastEditsIndexTime = DateTime.MinValue.ToString();
}
indexWriter.Commit(CreateCommitMetadata(lastEditsIndexTime, highestPackageKey, rangeToIndex.Count, "add"));
log.WriteLine("commit done");
}
log.WriteLine("end AddToIndex");
}
private static void AddPackagesToIndex(List<IndexDocumentData> indexDocumentData, Lucene.Net.Store.Directory directory, TextWriter log, PerfEventTracker perfTracker)
{
log.WriteLine("About to add {0} packages", indexDocumentData.Count);
for (int index = 0; index < indexDocumentData.Count; index += MaxDocumentsPerCommit)
{
int count = Math.Min(MaxDocumentsPerCommit, indexDocumentData.Count - index);
List<IndexDocumentData> rangeToIndex = indexDocumentData.GetRange(index, count);
AddToIndex(directory, rangeToIndex, log, perfTracker);
}
}
