static LuceneIndex() { System.IO.Directory.CreateDirectory(IndexDirectory); directory = new NRTCachingDirectory(new MMapDirectory(IndexDirectory), 256, 256); analyzer = new WordAnalyzer(LuceneVersion.LUCENE_48); writer = new IndexWriter(directory, new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer) { IndexDeletionPolicy = new KeepOnlyLastCommitDeletionPolicy(), MergedSegmentWarmer = new SimpleMergedSegmentWarmer(InfoStream.NO_OUTPUT), //UseCompoundFile = false, OpenMode = OpenMode.CREATE_OR_APPEND, UseReaderPooling = true, MergeScheduler = new TaskMergeScheduler(), MergePolicy = new TieredMergePolicy(), RAMBufferSizeMB = 1024, MaxBufferedDocs = 1024 }.SetInfoStream(InfoStream.NO_OUTPUT)); trackingWriter = new TrackingIndexWriter(writer); searcherKeeper = new SearcherManager(writer, false, null); reopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(trackingWriter, searcherKeeper, 3.0, 0.0) { Priority = ThreadPriority.BelowNormal, IsBackground = true }; reopenThread.Start(); new Thread(() => { while (true) { Thread.Sleep(30000); try { writer.Commit(); } catch {} } }) { IsBackground = true, Priority = ThreadPriority.BelowNormal }.Start(); }
// LUCENE-5461 public virtual void TestCRTReopen() { //test behaving badly //should be high enough int maxStaleSecs = 20; //build crap data just to store it. string s = " abcdefghijklmnopqrstuvwxyz "; char[] chars = s.ToCharArray(); StringBuilder builder = new StringBuilder(2048); for (int i = 0; i < 2048; i++) { builder.Append(chars[Random().Next(chars.Length)]); } string content = builder.ToString(); SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random())); config.SetIndexDeletionPolicy(sdp); config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND); IndexWriter iw = new IndexWriter(dir, config); SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory()); TrackingIndexWriter tiw = new TrackingIndexWriter(iw); ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0); controlledRealTimeReopenThread.SetDaemon(true); controlledRealTimeReopenThread.Start(); IList <Thread> commitThreads = new List <Thread>(); for (int i = 0; i < 500; i++) { if (i > 0 && i % 50 == 0) { Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw)); commitThread.Start(); commitThreads.Add(commitThread); } Document d = new Document(); d.Add(new TextField("count", i + "", Field.Store.NO)); d.Add(new TextField("content", content, Field.Store.YES)); long start = DateTime.Now.Millisecond; long l = tiw.AddDocument(d); controlledRealTimeReopenThread.WaitForGeneration(l); long wait = DateTime.Now.Millisecond - start; Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait); IndexSearcher searcher = sm.Acquire(); TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10); sm.Release(searcher); Assert.AreEqual(1, td.TotalHits); } foreach (Thread commitThread in commitThreads) { commitThread.Join(); } controlledRealTimeReopenThread.Dispose(); sm.Dispose(); iw.Dispose(); dir.Dispose(); }
private static BaseDirectoryWrapper WrapDirectory(Random random, Directory directory, bool bare) { if (Rarely(random)) { directory = new NRTCachingDirectory(directory, random.NextDouble(), random.NextDouble()); } if (Rarely(random)) { double maxMBPerSec = 10 + 5 * (random.NextDouble() - 0.5); if (LuceneTestCase.VERBOSE) { Console.WriteLine("LuceneTestCase: will rate limit output IndexOutput to " + maxMBPerSec + " MB/sec"); } RateLimitedDirectoryWrapper rateLimitedDirectoryWrapper = new RateLimitedDirectoryWrapper(directory); switch (random.Next(10)) { case 3: // sometimes rate limit on flush rateLimitedDirectoryWrapper.SetMaxWriteMBPerSec(maxMBPerSec, IOContext.UsageContext.FLUSH); break; case 2: // sometimes rate limit flush & merge rateLimitedDirectoryWrapper.SetMaxWriteMBPerSec(maxMBPerSec, IOContext.UsageContext.FLUSH); rateLimitedDirectoryWrapper.SetMaxWriteMBPerSec(maxMBPerSec, IOContext.UsageContext.MERGE); break; default: rateLimitedDirectoryWrapper.SetMaxWriteMBPerSec(maxMBPerSec, IOContext.UsageContext.MERGE); break; } directory = rateLimitedDirectoryWrapper; } if (bare) { BaseDirectoryWrapper @base = new BaseDirectoryWrapper(directory); // LUCENENET TODO CloseAfterSuite(new IDisposableDirectory(@base, SuiteFailureMarker)); return @base; } else { MockDirectoryWrapper mock = new MockDirectoryWrapper(random, directory); mock.Throttling = TEST_THROTTLING; // LUCENENET TODO CloseAfterSuite(new IDisposableDirectory(mock, SuiteFailureMarker)); return mock; } }
public virtual void TestCRTReopen() { //test behaving badly //should be high enough int maxStaleSecs = 20; //build crap data just to store it. string s = " abcdefghijklmnopqrstuvwxyz "; char[] chars = s.ToCharArray(); StringBuilder builder = new StringBuilder(2048); for (int i = 0; i < 2048; i++) { builder.Append(chars[Random.Next(chars.Length)]); } string content = builder.ToString(); SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128); IndexWriterConfig config = new IndexWriterConfig( #pragma warning disable 612, 618 Version.LUCENE_46, #pragma warning restore 612, 618 new MockAnalyzer(Random)); config.SetIndexDeletionPolicy(sdp); config.SetOpenMode(OpenMode.CREATE_OR_APPEND); IndexWriter iw = new IndexWriter(dir, config); SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory()); TrackingIndexWriter tiw = new TrackingIndexWriter(iw); ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0); controlledRealTimeReopenThread.IsBackground = (true); controlledRealTimeReopenThread.Start(); IList <ThreadJob> commitThreads = new JCG.List <ThreadJob>(); for (int i = 0; i < 500; i++) { if (i > 0 && i % 50 == 0) { ThreadJob commitThread = new RunnableAnonymousClass(this, sdp, dir, iw); commitThread.Start(); commitThreads.Add(commitThread); } Document d = new Document(); d.Add(new TextField("count", i + "", Field.Store.NO)); d.Add(new TextField("content", content, Field.Store.YES)); long start = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results long l = tiw.AddDocument(d); controlledRealTimeReopenThread.WaitForGeneration(l); long wait = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - start; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results assertTrue("waited too long for generation " + wait, wait < (maxStaleSecs * 1000)); IndexSearcher searcher = sm.Acquire(); TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10); sm.Release(searcher); assertEquals(1, td.TotalHits); } foreach (ThreadJob commitThread in commitThreads) { commitThread.Join(); } controlledRealTimeReopenThread.Dispose(); sm.Dispose(); iw.Dispose(); dir.Dispose(); }