private static void ApplyAdds(List <int> packageKeys, Func <int, IndexDocumentData> fetch, Lucene.Net.Store.Directory directory, TextWriter log, PerfEventTracker perfTracker, IEnumerable <FrameworkName> projectFxs) { log.WriteLine("ApplyAdds"); // Collect all the packages var packages = packageKeys.Select(k => fetch(k)); using (IndexWriter indexWriter = CreateIndexWriter(directory, false)) { IDictionary <string, string> commitUserData; var dirtyDocuments = new List <FacetedDocument>(); using (var reader = indexWriter.GetReader()) { commitUserData = reader.CommitUserData; foreach (var group in packages.GroupBy(p => p.Package.PackageRegistration.Id)) { var newDirtyDocs = DetermineDirtyDocuments(projectFxs, perfTracker, reader, group.Key, group); dirtyDocuments.AddRange(newDirtyDocs); } } WriteDirtyDocuments(dirtyDocuments, indexWriter, perfTracker); string lastEditsIndexTime = commitUserData["last-edits-index-time"]; if (lastEditsIndexTime == null) { // this should never happen but if it did Lucene would throw lastEditsIndexTime = DateTime.MinValue.ToString(); } log.WriteLine("Commit {0} adds", packageKeys.Count); indexWriter.Commit(PackageIndexing.CreateCommitMetadata(lastEditsIndexTime, packageKeys.Max(), packageKeys.Count, "add")); } }
public static void CreateNewEmptyIndex(Lucene.Net.Store.Directory directory) { using (IndexWriter indexWriter = CreateIndexWriter(directory, true)) { indexWriter.Commit(PackageIndexing.CreateCommitMetadata(DateTime.MinValue, 0, 0, "creation")); } }
private static void AddToIndex(Lucene.Net.Store.Directory directory, List <IndexDocumentData> rangeToIndex, TextWriter log, IEnumerable <FrameworkName> projectFxs, PerfEventTracker perfTracker) { log.WriteLine("begin AddToIndex"); int highestPackageKey = -1; var groups = rangeToIndex.GroupBy(d => d.Package.PackageRegistration.Id).ToList(); // Collect documents to change var dirtyDocs = new List <FacetedDocument>(); using (var reader = IndexReader.Open(directory, readOnly: true)) using (perfTracker.TrackEvent("CalculateChanges", "")) { foreach (var group in groups) { var newDirtyDocs = DetermineDirtyDocuments(projectFxs, perfTracker, reader, group.Key, group); // (Re-)Add any dirty documents to the index dirtyDocs.AddRange(newDirtyDocs); } } using (IndexWriter indexWriter = CreateIndexWriter(directory, create: false)) { WriteDirtyDocuments(dirtyDocs, indexWriter, perfTracker); highestPackageKey = rangeToIndex.Max(i => i.Package.Key); log.WriteLine("about to commit {0} packages", rangeToIndex.Count); IDictionary <string, string> commitUserData = indexWriter.GetReader().CommitUserData; string lastEditsIndexTime = commitUserData["last-edits-index-time"]; if (lastEditsIndexTime == null) { // this should never happen but if it did Lucene would throw lastEditsIndexTime = DateTime.MinValue.ToString(); } indexWriter.Commit(PackageIndexing.CreateCommitMetadata(lastEditsIndexTime, highestPackageKey, rangeToIndex.Count, "add")); log.WriteLine("commit done"); } log.WriteLine("end AddToIndex"); }
public override void Execute() { // Check for the frameworks list var frameworks = GetFrameworksList(); DateTime before = DateTime.Now; if (Force && StorageAccount != null && !string.IsNullOrEmpty(Container)) { AzureDirectoryManagement.ForceUnlockAzureDirectory(StorageAccount, Container); } Lucene.Net.Store.Directory directory = GetDirectory(); PackageIndexing.RebuildIndex(SqlConnectionString, directory, frameworks, Log); DateTime after = DateTime.Now; Log.WriteLine("duration = {0} seconds", (after - before).TotalSeconds); }
public override void Execute() { var manager = GetSearcherManager(); IDictionary <int, int> databaseChecksums = GalleryExport.FetchGalleryChecksums(SqlConnectionString); Log.WriteLine("fetched {0} keys from database", databaseChecksums.Count); Tuple <int, int> minMax = GalleryExport.FindMinMaxKey(databaseChecksums); Log.WriteLine("min = {0}, max = {1}", minMax.Item1, minMax.Item2); // For now, use the in-memory Searcher client. But eventually this will use the original Search Service call below IDictionary <int, int> index = ParseRangeResult( Searcher.KeyRangeQuery(manager, minMax.Item1, minMax.Item2)); Log.WriteLine("fetched {0} keys from index", index.Count); List <int> adds = new List <int>(); List <int> updates = new List <int>(); List <int> deletes = new List <int>(); SortIntoAddsUpdateDeletes(databaseChecksums, index, adds, updates, deletes); Log.WriteLine("{0} adds", adds.Count); Log.WriteLine("{0} updates", updates.Count); Log.WriteLine("{0} deletes", deletes.Count); if (adds.Count == 0 && updates.Count == 0) { return; } IDictionary <int, IEnumerable <string> > feeds = GalleryExport.GetFeedsByPackageRegistration(SqlConnectionString, Log, verbose: false); IDictionary <int, IndexDocumentData> packages = PackageIndexing.LoadDocumentData(SqlConnectionString, adds, updates, deletes, feeds, databaseChecksums, Log); Lucene.Net.Store.Directory directory = manager.Directory; Func <int, IndexDocumentData> packageFetcher = (key) => { IndexDocumentData knownDoc; if (packages.TryGetValue(key, out knownDoc)) { return(knownDoc); } else { // We're modifying a different document var pkgs = GalleryExport.GetPackages(SqlConnectionString, new List <int>() { key }, Log, verbose: false); var docs = PackageIndexing.MakeIndexDocumentData(pkgs, feeds, databaseChecksums); packages[key] = docs[0]; return(docs[0]); } }; var perfTracker = new PerfEventTracker(); PackageIndexing.UpdateIndex(WhatIf, adds, updates, deletes, packageFetcher, directory, Log, perfTracker, manager.Frameworks.Load()); }