private static void ApplyAdds(List <int> packageKeys, Func <int, IndexDocumentData> fetch, Lucene.Net.Store.Directory directory, TextWriter log, PerfEventTracker perfTracker, IEnumerable <FrameworkName> projectFxs)
{
log.WriteLine("ApplyAdds");
// Collect all the packages
var packages = packageKeys.Select(k => fetch(k));
using (IndexWriter indexWriter = CreateIndexWriter(directory, false))
{
IDictionary <string, string> commitUserData;
var dirtyDocuments = new List <FacetedDocument>();
using (var reader = indexWriter.GetReader())
{
commitUserData = reader.CommitUserData;
foreach (var group in packages.GroupBy(p => p.Package.PackageRegistration.Id))
{
var newDirtyDocs = DetermineDirtyDocuments(projectFxs, perfTracker, reader, group.Key, group);
dirtyDocuments.AddRange(newDirtyDocs);
}
}
WriteDirtyDocuments(dirtyDocuments, indexWriter, perfTracker);
string lastEditsIndexTime = commitUserData["last-edits-index-time"];
if (lastEditsIndexTime == null)
{
// this should never happen but if it did Lucene would throw
lastEditsIndexTime = DateTime.MinValue.ToString();
}
log.WriteLine("Commit {0} adds", packageKeys.Count);
indexWriter.Commit(PackageIndexing.CreateCommitMetadata(lastEditsIndexTime, packageKeys.Max(), packageKeys.Count, "add"));
}
}
public static void CreateNewEmptyIndex(Lucene.Net.Store.Directory directory)
{
using (IndexWriter indexWriter = CreateIndexWriter(directory, true))
{
indexWriter.Commit(PackageIndexing.CreateCommitMetadata(DateTime.MinValue, 0, 0, "creation"));
}
}
private static void AddToIndex(Lucene.Net.Store.Directory directory, List <IndexDocumentData> rangeToIndex, TextWriter log, IEnumerable <FrameworkName> projectFxs, PerfEventTracker perfTracker)
{
log.WriteLine("begin AddToIndex");
int highestPackageKey = -1;
var groups = rangeToIndex.GroupBy(d => d.Package.PackageRegistration.Id).ToList();
// Collect documents to change
var dirtyDocs = new List <FacetedDocument>();
using (var reader = IndexReader.Open(directory, readOnly: true))
using (perfTracker.TrackEvent("CalculateChanges", ""))
{
foreach (var group in groups)
{
var newDirtyDocs = DetermineDirtyDocuments(projectFxs, perfTracker, reader, group.Key, group);
// (Re-)Add any dirty documents to the index
dirtyDocs.AddRange(newDirtyDocs);
}
}
using (IndexWriter indexWriter = CreateIndexWriter(directory, create: false))
{
WriteDirtyDocuments(dirtyDocs, indexWriter, perfTracker);
highestPackageKey = rangeToIndex.Max(i => i.Package.Key);
log.WriteLine("about to commit {0} packages", rangeToIndex.Count);
IDictionary <string, string> commitUserData = indexWriter.GetReader().CommitUserData;
string lastEditsIndexTime = commitUserData["last-edits-index-time"];
if (lastEditsIndexTime == null)
{
// this should never happen but if it did Lucene would throw
lastEditsIndexTime = DateTime.MinValue.ToString();
}
indexWriter.Commit(PackageIndexing.CreateCommitMetadata(lastEditsIndexTime, highestPackageKey, rangeToIndex.Count, "add"));
log.WriteLine("commit done");
}
log.WriteLine("end AddToIndex");
}
public override void Execute()
{
// Check for the frameworks list
var frameworks = GetFrameworksList();
DateTime before = DateTime.Now;
if (Force && StorageAccount != null && !string.IsNullOrEmpty(Container))
{
AzureDirectoryManagement.ForceUnlockAzureDirectory(StorageAccount, Container);
}
Lucene.Net.Store.Directory directory = GetDirectory();
PackageIndexing.RebuildIndex(SqlConnectionString, directory, frameworks, Log);
DateTime after = DateTime.Now;
Log.WriteLine("duration = {0} seconds", (after - before).TotalSeconds);
}
public override void Execute()
{
var manager = GetSearcherManager();
IDictionary <int, int> databaseChecksums = GalleryExport.FetchGalleryChecksums(SqlConnectionString);
Log.WriteLine("fetched {0} keys from database", databaseChecksums.Count);
Tuple <int, int> minMax = GalleryExport.FindMinMaxKey(databaseChecksums);
Log.WriteLine("min = {0}, max = {1}", minMax.Item1, minMax.Item2);
// For now, use the in-memory Searcher client. But eventually this will use the original Search Service call below
IDictionary <int, int> index = ParseRangeResult(
Searcher.KeyRangeQuery(manager, minMax.Item1, minMax.Item2));
Log.WriteLine("fetched {0} keys from index", index.Count);
List <int> adds = new List <int>();
List <int> updates = new List <int>();
List <int> deletes = new List <int>();
SortIntoAddsUpdateDeletes(databaseChecksums, index, adds, updates, deletes);
Log.WriteLine("{0} adds", adds.Count);
Log.WriteLine("{0} updates", updates.Count);
Log.WriteLine("{0} deletes", deletes.Count);
if (adds.Count == 0 && updates.Count == 0)
{
return;
}
IDictionary <int, IEnumerable <string> > feeds = GalleryExport.GetFeedsByPackageRegistration(SqlConnectionString, Log, verbose: false);
IDictionary <int, IndexDocumentData> packages = PackageIndexing.LoadDocumentData(SqlConnectionString, adds, updates, deletes, feeds, databaseChecksums, Log);
Lucene.Net.Store.Directory directory = manager.Directory;
Func <int, IndexDocumentData> packageFetcher = (key) =>
{
IndexDocumentData knownDoc;
if (packages.TryGetValue(key, out knownDoc))
{
return(knownDoc);
}
else
{
// We're modifying a different document
var pkgs = GalleryExport.GetPackages(SqlConnectionString, new List <int>()
{
key
}, Log, verbose: false);
var docs = PackageIndexing.MakeIndexDocumentData(pkgs, feeds, databaseChecksums);
packages[key] = docs[0];
return(docs[0]);
}
};
var perfTracker = new PerfEventTracker();
PackageIndexing.UpdateIndex(WhatIf, adds, updates, deletes, packageFetcher, directory, Log, perfTracker, manager.Frameworks.Load());
}