/*
         * LUCENE-3528 - NRTManager hangs in certain situations
         */
        public virtual void TestThreadStarvationNoDeleteNRTReader()
        {
            IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));

            conf.SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
            Directory      d      = NewDirectory();
            CountDownLatch latch  = new CountDownLatch(1);
            CountDownLatch signal = new CountDownLatch(1);

            LatchedIndexWriter  _writer = new LatchedIndexWriter(d, conf, latch, signal);
            TrackingIndexWriter writer  = new TrackingIndexWriter(_writer);
            SearcherManager     manager = new SearcherManager(_writer, false, null);
            Document            doc     = new Document();

            doc.Add(NewTextField("test", "test", Field.Store.YES));
            writer.AddDocument(doc);
            manager.MaybeRefresh();
            ThreadClass t = new ThreadAnonymousInnerClassHelper(this, latch, signal, writer, manager);

            t.Start();
            _writer.WaitAfterUpdate = true;                                    // wait in addDocument to let some reopens go through
            long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen

            Assert.IsFalse(manager.SearcherCurrent);                           // false since there is a delete in the queue

            IndexSearcher searcher = manager.Acquire();

            try
            {
                Assert.AreEqual(2, searcher.IndexReader.NumDocs());
            }
            finally
            {
                manager.Release(searcher);
            }
            ControlledRealTimeReopenThread <IndexSearcher> thread = new ControlledRealTimeReopenThread <IndexSearcher>(writer, manager, 0.01, 0.01);

            thread.Start(); // start reopening
            if (VERBOSE)
            {
                Console.WriteLine("waiting now for generation " + lastGen);
            }

            AtomicBoolean finished = new AtomicBoolean(false);
            ThreadClass   waiter   = new ThreadAnonymousInnerClassHelper2(this, lastGen, thread, finished);

            waiter.Start();
            manager.MaybeRefresh();
            waiter.Join(1000);
            if (!finished.Get())
            {
                waiter.Interrupt();
                Assert.Fail("thread deadlocked on waitForGeneration");
            }
            thread.Dispose();
            thread.Join();
            IOUtils.Close(manager, _writer, d);
        }
        // LUCENE-5461
        public virtual void TestCRTReopen()
        {
            //test behaving badly

            //should be high enough
            int maxStaleSecs = 20;

            //build crap data just to store it.
            string s = "        abcdefghijklmnopqrstuvwxyz     ";

            char[]        chars   = s.ToCharArray();
            StringBuilder builder = new StringBuilder(2048);

            for (int i = 0; i < 2048; i++)
            {
                builder.Append(chars[Random().Next(chars.Length)]);
            }
            string content = builder.ToString();

            SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
            Directory         dir      = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
            IndexWriterConfig config   = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random()));

            config.SetIndexDeletionPolicy(sdp);
            config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND);
            IndexWriter         iw  = new IndexWriter(dir, config);
            SearcherManager     sm  = new SearcherManager(iw, true, new SearcherFactory());
            TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
            ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0);

            controlledRealTimeReopenThread.SetDaemon(true);
            controlledRealTimeReopenThread.Start();

            IList <Thread> commitThreads = new List <Thread>();

            for (int i = 0; i < 500; i++)
            {
                if (i > 0 && i % 50 == 0)
                {
                    Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw));
                    commitThread.Start();
                    commitThreads.Add(commitThread);
                }
                Document d = new Document();
                d.Add(new TextField("count", i + "", Field.Store.NO));
                d.Add(new TextField("content", content, Field.Store.YES));
                long start = DateTime.Now.Millisecond;
                long l     = tiw.AddDocument(d);
                controlledRealTimeReopenThread.WaitForGeneration(l);
                long wait = DateTime.Now.Millisecond - start;
                Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait);
                IndexSearcher searcher = sm.Acquire();
                TopDocs       td       = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
                sm.Release(searcher);
                Assert.AreEqual(1, td.TotalHits);
            }

            foreach (Thread commitThread in commitThreads)
            {
                commitThread.Join();
            }

            controlledRealTimeReopenThread.Dispose();
            sm.Dispose();
            iw.Dispose();
            dir.Dispose();
        }
Ejemplo n.º 3
0
        public void TestMultithreadedWaitForGeneration()
        {
            Thread CreateWorker(int threadNum, ControlledRealTimeReopenThread <IndexSearcher> controlledReopen, long generation,
                                SearcherManager searcherManager, List <ThreadOutput> outputList)
            {
                ThreadStart threadStart = delegate
                {
                    Stopwatch stopwatch = Stopwatch.StartNew();
                    controlledReopen.WaitForGeneration(generation);
                    stopwatch.Stop();
                    double milliSecsWaited = stopwatch.Elapsed.TotalMilliseconds;

                    int           numRecs       = 0;
                    IndexSearcher indexSearcher = searcherManager.Acquire();
                    try
                    {
                        TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
                        numRecs = topDocs.TotalHits;
                    }
                    finally
                    {
                        searcherManager.Release(indexSearcher);
                    }

                    lock (outputList)
                    {
                        outputList.Add(new ThreadOutput {
                            ThreadNum = threadNum, NumRecs = numRecs, MilliSecsWaited = milliSecsWaited
                        });
                    }
                };

                return(new Thread(threadStart));
            }

            int threadCount = 3;
            List <ThreadOutput> outputList = new List <ThreadOutput>();

            RAMDirectory        indexDir         = new RAMDirectory();
            Analyzer            standardAnalyzer = new StandardAnalyzer(TEST_VERSION_CURRENT);
            IndexWriterConfig   indexConfig      = new IndexWriterConfig(TEST_VERSION_CURRENT, standardAnalyzer);
            IndexWriter         indexWriter      = new IndexWriter(indexDir, indexConfig);
            TrackingIndexWriter trackingWriter   = new TrackingIndexWriter(indexWriter);

            //Add two documents
            Document doc = new Document();

            doc.Add(new Int32Field("id", 1, Field.Store.YES));
            doc.Add(new StringField("name", "Doc1", Field.Store.YES));
            long generation = trackingWriter.AddDocument(doc);

            doc.Add(new Int32Field("id", 2, Field.Store.YES));
            doc.Add(new StringField("name", "Doc3", Field.Store.YES));
            generation = trackingWriter.AddDocument(doc);

            SearcherManager searcherManager = new SearcherManager(indexWriter, applyAllDeletes: true, null);

            //Reopen SearcherManager every 2 secs via background thread if no thread waiting for newer generation.
            //Reopen SearcherManager after .2 secs if another thread IS waiting on a newer generation.
            double maxRefreshSecs = 2.0;
            double minRefreshSecs = .2;
            var    controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(trackingWriter, searcherManager, maxRefreshSecs, minRefreshSecs);

            //Start() will start a seperate thread that will invoke the object's Run(). However,
            //calling Run() directly would execute that code on the current thread rather then a new thread
            //which would defeat the purpose of using controlledRealTimeReopenThread. This aspect of the API
            //is not as intuitive as it could be. ie. Call Start() not Run().
            controlledRealTimeReopenThread.IsBackground = true;                     //Set as a background thread
            controlledRealTimeReopenThread.Name         = "Controlled Real Time Reopen Thread";
            controlledRealTimeReopenThread.Priority     = (ThreadPriority)Math.Min((int)Thread.CurrentThread.Priority + 2, (int)ThreadPriority.Highest);
            controlledRealTimeReopenThread.Start();


            //Create the threads for doing searchers
            List <Thread> threadList = new List <Thread>();

            for (int i = 1; i <= threadCount; i++)
            {
                threadList.Add(CreateWorker(i, controlledRealTimeReopenThread, generation, searcherManager, outputList));
            }

            //Start all the threads
            foreach (Thread thread in threadList)
            {
                thread.Start();
            }

            //wait for the threads to finish.
            foreach (Thread thread in threadList)
            {
                thread.Join();                          //will wait here until the thread terminates.
            }

            //Now make sure that no thread waited longer then our min refresh time
            //plus a small fudge factor. Also verify that all threads resported back and
            //each saw 2 records.

            //Verify all threads reported back a result.
            assertEquals(threadCount, outputList.Count);

            int millisecsPerSec = 1000;

            foreach (ThreadOutput output in outputList)
            {
                //Verify the thread saw exactly 2 docs
                assertEquals(2, output.NumRecs);

                //Verify the thread wait time was around what was expected.
                Assert.True(output.MilliSecsWaited <= (minRefreshSecs * millisecsPerSec) + 30);   //30ms is fudged factor to account for call overhead
            }

            controlledRealTimeReopenThread.Dispose();                       //will kill and join to the thread
            Assert.False(controlledRealTimeReopenThread.IsAlive);           //to prove that Dispose really does kill the thread.

            searcherManager.Dispose();
            indexWriter.Dispose();
            indexDir.Dispose();
        }
        public virtual void TestCRTReopen()
        {
            //test behaving badly

            //should be high enough
            int maxStaleSecs = 20;

            //build crap data just to store it.
            string s = "        abcdefghijklmnopqrstuvwxyz     ";

            char[]        chars   = s.ToCharArray();
            StringBuilder builder = new StringBuilder(2048);

            for (int i = 0; i < 2048; i++)
            {
                builder.Append(chars[Random.Next(chars.Length)]);
            }
            string content = builder.ToString();

            SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
            Directory         dir      = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
            IndexWriterConfig config   = new IndexWriterConfig(
#pragma warning disable 612, 618
                Version.LUCENE_46,
#pragma warning restore 612, 618
                new MockAnalyzer(Random));

            config.SetIndexDeletionPolicy(sdp);
            config.SetOpenMode(OpenMode.CREATE_OR_APPEND);
            IndexWriter         iw  = new IndexWriter(dir, config);
            SearcherManager     sm  = new SearcherManager(iw, true, new SearcherFactory());
            TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
            ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread =
                new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0);

            controlledRealTimeReopenThread.IsBackground = (true);
            controlledRealTimeReopenThread.Start();

            IList <ThreadJob> commitThreads = new JCG.List <ThreadJob>();

            for (int i = 0; i < 500; i++)
            {
                if (i > 0 && i % 50 == 0)
                {
                    ThreadJob commitThread = new RunnableAnonymousClass(this, sdp, dir, iw);
                    commitThread.Start();
                    commitThreads.Add(commitThread);
                }
                Document d = new Document();
                d.Add(new TextField("count", i + "", Field.Store.NO));
                d.Add(new TextField("content", content, Field.Store.YES));
                long start = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
                long l     = tiw.AddDocument(d);
                controlledRealTimeReopenThread.WaitForGeneration(l);
                long wait = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - start; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
                assertTrue("waited too long for generation " + wait, wait < (maxStaleSecs * 1000));
                IndexSearcher searcher = sm.Acquire();
                TopDocs       td       = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
                sm.Release(searcher);
                assertEquals(1, td.TotalHits);
            }

            foreach (ThreadJob commitThread in commitThreads)
            {
                commitThread.Join();
            }

            controlledRealTimeReopenThread.Dispose();
            sm.Dispose();
            iw.Dispose();
            dir.Dispose();
        }
Ejemplo n.º 5
0
        public void TestStraightForwardDemonstration()
        {
            RAMDirectory indexDir = new RAMDirectory();

            Analyzer            standardAnalyzer = new StandardAnalyzer(TEST_VERSION_CURRENT);
            IndexWriterConfig   indexConfig      = new IndexWriterConfig(TEST_VERSION_CURRENT, standardAnalyzer);
            IndexWriter         indexWriter      = new IndexWriter(indexDir, indexConfig);
            TrackingIndexWriter trackingWriter   = new TrackingIndexWriter(indexWriter);

            Document doc = new Document();

            doc.Add(new Int32Field("id", 1, Field.Store.YES));
            doc.Add(new StringField("name", "Doc1", Field.Store.YES));
            trackingWriter.AddDocument(doc);

            SearcherManager searcherManager = new SearcherManager(indexWriter, applyAllDeletes: true, null);

            //Reopen SearcherManager every 1 secs via background thread if no thread waiting for newer generation.
            //Reopen SearcherManager after .2 secs if another thread IS waiting on a newer generation.
            var controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(trackingWriter, searcherManager, 1, 0.2);

            //Start() will start a seperate thread that will invoke the object's Run(). However,
            //calling Run() directly would execute that code on the current thread rather then a new thread
            //which would defeat the purpose of using controlledRealTimeReopenThread. This aspect of the API
            //is not as intuitive as it could be. ie. Call Start() not Run().
            controlledRealTimeReopenThread.IsBackground = true;                     //Set as a background thread
            controlledRealTimeReopenThread.Name         = "Controlled Real Time Reopen Thread";
            controlledRealTimeReopenThread.Priority     = (ThreadPriority)Math.Min((int)Thread.CurrentThread.Priority + 2, (int)ThreadPriority.Highest);
            controlledRealTimeReopenThread.Start();

            //An indexSearcher only sees Doc1
            IndexSearcher indexSearcher = searcherManager.Acquire();

            try
            {
                TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
                assertEquals(1, topDocs.TotalHits);             //There is only one doc
            }
            finally
            {
                searcherManager.Release(indexSearcher);
            }

            //Add a 2nd document
            doc = new Document();
            doc.Add(new Int32Field("id", 2, Field.Store.YES));
            doc.Add(new StringField("name", "Doc2", Field.Store.YES));
            trackingWriter.AddDocument(doc);

            //Demonstrate that we can only see the first doc because we haven't
            //waited 1 sec or called WaitForGeneration
            indexSearcher = searcherManager.Acquire();
            try
            {
                TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
                assertEquals(1, topDocs.TotalHits);             //Can see both docs due to auto refresh after 1.1 secs
            }
            finally
            {
                searcherManager.Release(indexSearcher);
            }


            //Demonstrate that we can see both docs after we wait a little more
            //then 1 sec so that controlledRealTimeReopenThread max interval is exceeded
            //and it calls MaybeRefresh
            Thread.Sleep(1100);     //wait 1.1 secs as ms
            indexSearcher = searcherManager.Acquire();
            try
            {
                TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
                assertEquals(2, topDocs.TotalHits);             //Can see both docs due to auto refresh after 1.1 secs
            }
            finally
            {
                searcherManager.Release(indexSearcher);
            }


            //Add a 3rd document
            doc = new Document();
            doc.Add(new Int32Field("id", 3, Field.Store.YES));
            doc.Add(new StringField("name", "Doc3", Field.Store.YES));
            long generation = trackingWriter.AddDocument(doc);

            //Demonstrate that if we call WaitForGeneration our wait will be
            // .2 secs or less (the min interval we set earlier) and then we will
            //see all 3 documents.
            Stopwatch stopwatch = Stopwatch.StartNew();

            controlledRealTimeReopenThread.WaitForGeneration(generation);
            stopwatch.Stop();
            assertTrue(stopwatch.Elapsed.TotalMilliseconds <= 200 + 30);   //30ms is fudged factor to account for call overhead.

            indexSearcher = searcherManager.Acquire();
            try
            {
                TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
                assertEquals(3, topDocs.TotalHits);             //Can see both docs due to auto refresh after 1.1 secs
            }
            finally
            {
                searcherManager.Release(indexSearcher);
            }

            controlledRealTimeReopenThread.Dispose();
            searcherManager.Dispose();
            indexWriter.Dispose();
            indexDir.Dispose();
        }