public static GetLastCommitGeneration ( Directory directory ) : long | ||
directory | Directory | -- directory to search for the latest segments_N file |
return | long |
public virtual void TestKeepLastNDeletionPolicyWithCreates() { const int N = 10; for (int pass = 0; pass < 2; pass++) { bool useCompoundFile = (pass % 2) != 0; Directory dir = NewDirectory(); IndexWriterConfig conf = (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetIndexDeletionPolicy(new KeepLastNDeletionPolicy(this, N)).SetMaxBufferedDocs(10); MergePolicy mp = conf.MergePolicy; mp.NoCFSRatio = useCompoundFile ? 1.0 : 0.0; IndexWriter writer = new IndexWriter(dir, conf); KeepLastNDeletionPolicy policy = (KeepLastNDeletionPolicy)writer.Config.IndexDeletionPolicy; writer.Dispose(); Term searchTerm = new Term("content", "aaa"); Query query = new TermQuery(searchTerm); for (int i = 0; i < N + 1; i++) { conf = (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetIndexDeletionPolicy(policy).SetMaxBufferedDocs(10); mp = conf.MergePolicy; mp.NoCFSRatio = useCompoundFile ? 1.0 : 0.0; writer = new IndexWriter(dir, conf); policy = (KeepLastNDeletionPolicy)writer.Config.IndexDeletionPolicy; for (int j = 0; j < 17; j++) { AddDocWithID(writer, i * (N + 1) + j); } // this is a commit writer.Dispose(); conf = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))).SetIndexDeletionPolicy(policy).SetMergePolicy(NoMergePolicy.COMPOUND_FILES); writer = new IndexWriter(dir, conf); policy = (KeepLastNDeletionPolicy)writer.Config.IndexDeletionPolicy; writer.DeleteDocuments(new Term("id", "" + (i * (N + 1) + 3))); // this is a commit writer.Dispose(); IndexReader reader = DirectoryReader.Open(dir); IndexSearcher searcher = NewSearcher(reader); ScoreDoc[] hits = searcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(16, hits.Length); reader.Dispose(); writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetIndexDeletionPolicy(policy)); policy = (KeepLastNDeletionPolicy)writer.Config.IndexDeletionPolicy; // this will not commit: there are no changes // pending because we opened for "create": writer.Dispose(); } Assert.AreEqual(3 * (N + 1) + 1, policy.NumOnInit); Assert.AreEqual(3 * (N + 1) + 1, policy.NumOnCommit); IndexReader rwReader = DirectoryReader.Open(dir); IndexSearcher searcher_ = NewSearcher(rwReader); ScoreDoc[] hits_ = searcher_.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(0, hits_.Length); // Simplistic check: just verify only the past N segments_N's still // exist, and, I can open a reader on each: long gen = SegmentInfos.GetLastCommitGeneration(dir); dir.DeleteFile(IndexFileNames.SEGMENTS_GEN); int expectedCount = 0; rwReader.Dispose(); for (int i = 0; i < N + 1; i++) { try { IndexReader reader = DirectoryReader.Open(dir); // Work backwards in commits on what the expected // count should be. searcher_ = NewSearcher(reader); hits_ = searcher_.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(expectedCount, hits_.Length); if (expectedCount == 0) { expectedCount = 16; } else if (expectedCount == 16) { expectedCount = 17; } else if (expectedCount == 17) { expectedCount = 0; } reader.Dispose(); if (i == N) { Assert.Fail("should have failed on commits before last " + N); } } catch (IOException /*e*/) { if (i != N) { throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details) } } if (i < N) { dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); } gen--; } dir.Dispose(); } }
public virtual void TestKeepAllDeletionPolicy() { for (int pass = 0; pass < 2; pass++) { if (VERBOSE) { Console.WriteLine("TEST: cycle pass="******"TEST: open writer for forceMerge"); } writer = new IndexWriter(dir, conf); policy = (KeepAllDeletionPolicy)writer.Config.IndexDeletionPolicy; writer.ForceMerge(1); writer.Dispose(); } Assert.AreEqual(needsMerging ? 2 : 1, policy.NumOnInit); // If we are not auto committing then there should // be exactly 2 commits (one per close above): Assert.AreEqual(1 + (needsMerging ? 1 : 0), policy.NumOnCommit); // Test listCommits ICollection <IndexCommit> commits = DirectoryReader.ListCommits(dir); // 2 from closing writer Assert.AreEqual(1 + (needsMerging ? 1 : 0), commits.Count); // Make sure we can open a reader on each commit: foreach (IndexCommit commit in commits) { IndexReader r = DirectoryReader.Open(commit); r.Dispose(); } // Simplistic check: just verify all segments_N's still // exist, and, I can open a reader on each: dir.DeleteFile(IndexFileNames.SEGMENTS_GEN); long gen = SegmentInfos.GetLastCommitGeneration(dir); while (gen > 0) { IndexReader reader = DirectoryReader.Open(dir); reader.Dispose(); dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); gen--; if (gen > 0) { // Now that we've removed a commit point, which // should have orphan'd at least one index file. // Open & close a writer and assert that it // actually removed something: int preCount = dir.ListAll().Length; writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetIndexDeletionPolicy(policy)); writer.Dispose(); int postCount = dir.ListAll().Length; Assert.IsTrue(postCount < preCount); } } dir.Dispose(); } }
public virtual void TestKeepLastNDeletionPolicy() { const int N = 5; for (int pass = 0; pass < 2; pass++) { bool useCompoundFile = (pass % 2) != 0; Directory dir = NewDirectory(); KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N); for (int j = 0; j < N + 1; j++) { IndexWriterConfig conf = (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetIndexDeletionPolicy(policy).SetMaxBufferedDocs(10); MergePolicy mp = conf.MergePolicy; mp.NoCFSRatio = useCompoundFile ? 1.0 : 0.0; IndexWriter writer = new IndexWriter(dir, conf); policy = (KeepLastNDeletionPolicy)writer.Config.IndexDeletionPolicy; for (int i = 0; i < 17; i++) { AddDoc(writer); } writer.ForceMerge(1); writer.Dispose(); } Assert.IsTrue(policy.NumDelete > 0); Assert.AreEqual(N + 1, policy.NumOnInit); Assert.AreEqual(N + 1, policy.NumOnCommit); // Simplistic check: just verify only the past N segments_N's still // exist, and, I can open a reader on each: dir.DeleteFile(IndexFileNames.SEGMENTS_GEN); long gen = SegmentInfos.GetLastCommitGeneration(dir); for (int i = 0; i < N + 1; i++) { try { IndexReader reader = DirectoryReader.Open(dir); reader.Dispose(); if (i == N) { Assert.Fail("should have failed on commits prior to last " + N); } } catch (IOException /*e*/) { if (i != N) { throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details) } } if (i < N) { dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); } gen--; } dir.Dispose(); } }
public virtual void TestExpirationTimeDeletionPolicy() { const double SECONDS = 2.0; Directory dir = NewDirectory(); IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetIndexDeletionPolicy(new ExpirationTimeDeletionPolicy(this, dir, SECONDS)); MergePolicy mp = conf.MergePolicy; mp.NoCFSRatio = 1.0; IndexWriter writer = new IndexWriter(dir, conf); ExpirationTimeDeletionPolicy policy = (ExpirationTimeDeletionPolicy)writer.Config.IndexDeletionPolicy; IDictionary <string, string> commitData = new Dictionary <string, string>(); commitData["commitTime"] = Convert.ToString(Environment.TickCount); writer.SetCommitData(commitData); writer.Commit(); writer.Dispose(); long lastDeleteTime = 0; int targetNumDelete = TestUtil.NextInt32(Random, 1, 5); while (policy.NumDelete < targetNumDelete) { // Record last time when writer performed deletes of // past commits lastDeleteTime = Environment.TickCount; conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetIndexDeletionPolicy(policy); mp = conf.MergePolicy; mp.NoCFSRatio = 1.0; writer = new IndexWriter(dir, conf); policy = (ExpirationTimeDeletionPolicy)writer.Config.IndexDeletionPolicy; for (int j = 0; j < 17; j++) { AddDoc(writer); } commitData = new Dictionary <string, string>(); commitData["commitTime"] = Convert.ToString(Environment.TickCount); writer.SetCommitData(commitData); writer.Commit(); writer.Dispose(); Thread.Sleep((int)(1000.0 * (SECONDS / 5.0))); } // Then simplistic check: just verify that the // segments_N's that still exist are in fact within SECONDS // seconds of the last one's mod time, and, that I can // open a reader on each: long gen = SegmentInfos.GetLastCommitGeneration(dir); string fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen); dir.DeleteFile(IndexFileNames.SEGMENTS_GEN); bool oneSecondResolution = true; while (gen > 0) { try { IndexReader reader = DirectoryReader.Open(dir); reader.Dispose(); fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen); // if we are on a filesystem that seems to have only // 1 second resolution, allow +1 second in commit // age tolerance: SegmentInfos sis = new SegmentInfos(); sis.Read(dir, fileName); long modTime = Convert.ToInt64(sis.UserData["commitTime"]); oneSecondResolution &= (modTime % 1000) == 0; long leeway = (long)((SECONDS + (oneSecondResolution ? 1.0 : 0.0)) * 1000); Assert.IsTrue(lastDeleteTime - modTime <= leeway, "commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted "); } #pragma warning disable 168 catch (IOException e) #pragma warning restore 168 { // OK break; } dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); gen--; } dir.Dispose(); }