private static void ProcessCompatibleVersion(string packageId, PerfEventTracker perfTracker, IDictionary <string, FacetedDocument> candidateNewFacets, FacetedDocument doc, FrameworkName projectFx)
{
using (perfTracker.TrackEvent("ProcessCompatibleVersion", "{0} v{1} (fx:{2})", packageId, doc.Version, projectFx))
{
// Add compatible facet
doc.AddFacet(Facets.Compatible(projectFx));
// If listed, process it against latest versions
if (doc.Data.Package.Listed)
{
// Check it against the current latest prerelease and swap latests if necessary
string latestPreFacet = Facets.LatestPrereleaseVersion(projectFx);
string latestStableFacet = Facets.LatestStableVersion(projectFx);
if (!candidateNewFacets.ContainsKey(latestPreFacet))
{
candidateNewFacets[latestPreFacet] = doc;
}
// If this package is a stable version, do the same for latest stable
if (String.IsNullOrEmpty(doc.Version.SpecialVersion) && !candidateNewFacets.ContainsKey(latestStableFacet))
{
candidateNewFacets[latestStableFacet] = doc;
}
}
}
}
// this function will incrementally build an index from the gallery using a high water mark stored in the commit metadata
// this function is useful for building a fresh index as in that case it is more efficient than diff-ing approach
public static void RebuildIndex(string sqlConnectionString, Lucene.Net.Store.Directory directory, FrameworksList frameworks, TextWriter log = null, PerfEventTracker perfTracker = null)
{
perfTracker = perfTracker ?? new PerfEventTracker();
log = log ?? DefaultTraceWriter;
Stopwatch sw = new Stopwatch();
sw.Start();
using (perfTracker.TrackEvent("RebuildIndex", String.Empty))
{
// Empty the index, we're rebuilding
CreateNewEmptyIndex(directory);
var projectFxs = frameworks.Load();
log.WriteLine("get curated feeds by PackageRegistration");
IDictionary <int, IEnumerable <string> > feeds = GalleryExport.GetFeedsByPackageRegistration(sqlConnectionString, log, verbose: false);
int highestPackageKey = 0;
while (true)
{
log.WriteLine("get the checksums from the gallery");
IDictionary <int, int> checksums = GalleryExport.FetchGalleryChecksums(sqlConnectionString, highestPackageKey);
log.WriteLine("get packages from gallery where the Package.Key > {0}", highestPackageKey);
List <Package> packages = GalleryExport.GetPublishedPackagesSince(sqlConnectionString, highestPackageKey, log, verbose: false);
if (packages.Count == 0)
{
break;
}
log.WriteLine("associate the feeds and checksum data with each packages");
List <IndexDocumentData> indexDocumentData = MakeIndexDocumentData(packages, feeds, checksums);
highestPackageKey = indexDocumentData.Max(d => d.Package.Key);
AddPackagesToIndex(indexDocumentData, directory, log, projectFxs, perfTracker);
// Summarize performance
// (Save some time by not bothering if the log is "null")
if (!ReferenceEquals(TextWriter.Null, log) && !ReferenceEquals(PerfEventTracker.Null, perfTracker))
{
SummarizePerf(log, perfTracker);
}
}
}
SummarizePerf(log, perfTracker);
sw.Stop();
log.WriteLine("all done, took {0}", sw.Elapsed);
}
private static IEnumerable <FacetedDocument> DetermineDirtyDocuments(IEnumerable <FrameworkName> projectFxs, PerfEventTracker perfTracker, IndexReader reader, string id, IEnumerable <IndexDocumentData> data)
{
using (perfTracker.TrackEvent("Processdata", id))
{
// Get all documents matching the ID of this data.
var documents = CollectExistingDocuments(perfTracker, reader, id);
// Add the new documents
using (perfTracker.TrackEvent("CreateNewDocuments", id))
{
foreach (var package in data)
{
documents.Add(new FacetedDocument(package));
}
}
// Process the facets
UpdateFacets(id, documents, projectFxs, perfTracker);
return(documents.Where(d => d.Dirty));
}
}
private static void WriteDirtyDocuments(List <FacetedDocument> dirtyDocs, IndexWriter indexWriter, PerfEventTracker perfTracker)
{
// Delete dirty documents and flush
foreach (var dirtyDoc in dirtyDocs.Where(d => d.Dirty))
{
indexWriter.DeleteDocuments(dirtyDoc.GetQuery());
}
using (perfTracker.TrackEvent("FlushingDeletes", ""))
{
indexWriter.Flush(triggerMerge: false, flushDocStores: true, flushDeletes: true);
}
// (Re-)add dirty documents
foreach (var dirtyDoc in dirtyDocs)
{
using (perfTracker.TrackEvent("AddDocument", "{0} v{1}", dirtyDoc.Id, dirtyDoc.Version))
{
indexWriter.AddDocument(CreateLuceneDocument(dirtyDoc));
}
}
}
private static List <FacetedDocument> CollectExistingDocuments(PerfEventTracker perfTracker, IndexReader reader, string id)
{
var docs = reader.TermDocs(new Term("Id", id.ToLowerInvariant()));
var documents = new List <FacetedDocument>();
using (perfTracker.TrackEvent("GetExistingDocuments", id))
{
while (docs.Next())
{
documents.Add(new FacetedDocument(reader.Document(docs.Doc)));
}
}
return(documents);
}
private static void AddToIndex(Lucene.Net.Store.Directory directory, List <IndexDocumentData> rangeToIndex, TextWriter log, IEnumerable <FrameworkName> projectFxs, PerfEventTracker perfTracker)
{
log.WriteLine("begin AddToIndex");
int highestPackageKey = -1;
var groups = rangeToIndex.GroupBy(d => d.Package.PackageRegistration.Id).ToList();
// Collect documents to change
var dirtyDocs = new List <FacetedDocument>();
using (var reader = IndexReader.Open(directory, readOnly: true))
using (perfTracker.TrackEvent("CalculateChanges", ""))
{
foreach (var group in groups)
{
var newDirtyDocs = DetermineDirtyDocuments(projectFxs, perfTracker, reader, group.Key, group);
// (Re-)Add any dirty documents to the index
dirtyDocs.AddRange(newDirtyDocs);
}
}
using (IndexWriter indexWriter = CreateIndexWriter(directory, create: false))
{
WriteDirtyDocuments(dirtyDocs, indexWriter, perfTracker);
highestPackageKey = rangeToIndex.Max(i => i.Package.Key);
log.WriteLine("about to commit {0} packages", rangeToIndex.Count);
IDictionary <string, string> commitUserData = indexWriter.GetReader().CommitUserData;
string lastEditsIndexTime = commitUserData["last-edits-index-time"];
if (lastEditsIndexTime == null)
{
// this should never happen but if it did Lucene would throw
lastEditsIndexTime = DateTime.MinValue.ToString();
}
indexWriter.Commit(PackageIndexing.CreateCommitMetadata(lastEditsIndexTime, highestPackageKey, rangeToIndex.Count, "add"));
log.WriteLine("commit done");
}
log.WriteLine("end AddToIndex");
}
private static void UpdateFacets(string packageId, IList <FacetedDocument> documents, IEnumerable <FrameworkName> projectFxs, PerfEventTracker perfTracker)
{
using (perfTracker.TrackEvent("UpdateFacets", "{0} ({1} items)", packageId, documents.Count))
{
// Collect all the current latest versions into dictionaries
IDictionary <string, List <FacetedDocument> > existingFacets = new Dictionary <string, List <FacetedDocument> >(StringComparer.OrdinalIgnoreCase);
using (perfTracker.TrackEvent("FindExistingFacets", packageId))
{
foreach (var document in documents.Where(d => !d.IsNew))
{
foreach (var projectFx in projectFxs)
{
AddToExistingFacetsList(existingFacets, document, projectFx, Facets.LatestStableVersion(projectFx));
AddToExistingFacetsList(existingFacets, document, projectFx, Facets.LatestPrereleaseVersion(projectFx));
}
}
}
IDictionary <string, FacetedDocument> candidateNewFacets = new Dictionary <string, FacetedDocument>();
// Process the new documents
var newDocs = documents.Where(d => d.IsNew).OrderByDescending(d => d.Version).ToList();
documents = null; // Done with the master list of all documents
using (perfTracker.TrackEvent("DetermineNewLatestVersions", packageId))
{
foreach (var doc in newDocs)
{
if (!String.IsNullOrEmpty(doc.Version.SpecialVersion))
{
doc.AddFacet(Facets.PrereleaseVersion);
}
if (doc.Data.Package.Listed)
{
doc.AddFacet(Facets.Listed);
}
var packageFxs = doc.Data.Package.SupportedFrameworks
.Select(fx =>
{
using (perfTracker.TrackEvent("ParseFrameworkName", fx.TargetFramework))
{
return(VersionUtility.ParseFrameworkName(fx.TargetFramework));
}
})
.ToList();
// Process each target framework
foreach (var projectFx in projectFxs)
{
if (projectFx == FrameworksList.AnyFramework || VersionUtility.IsCompatible(projectFx, packageFxs))
{
ProcessCompatibleVersion(packageId, perfTracker, candidateNewFacets, doc, projectFx);
}
}
}
}
// Adjust facets as needed
using (perfTracker.TrackEvent("AdjustProjectFxes", packageId))
{
foreach (var projectFx in projectFxs)
{
using (perfTracker.TrackEvent("AdjustProjectFx", "{0} ({1})", packageId, projectFx.FullName))
{
UpdateLatestVersionFacet(existingFacets, candidateNewFacets, Facets.LatestStableVersion(projectFx));
UpdateLatestVersionFacet(existingFacets, candidateNewFacets, Facets.LatestPrereleaseVersion(projectFx));
}
}
}
}
}