public virtual void TestCRTReopen()
{
//test behaving badly
//should be high enough
int maxStaleSecs = 20;
//build crap data just to store it.
string s = " abcdefghijklmnopqrstuvwxyz ";
char[] chars = s.ToCharArray();
StringBuilder builder = new StringBuilder(2048);
for (int i = 0; i < 2048; i++)
{
builder.Append(chars[Random.Next(chars.Length)]);
}
string content = builder.ToString();
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
IndexWriterConfig config = new IndexWriterConfig(
#pragma warning disable 612, 618
Version.LUCENE_46,
#pragma warning restore 612, 618
new MockAnalyzer(Random));
config.SetIndexDeletionPolicy(sdp);
config.SetOpenMode(OpenMode.CREATE_OR_APPEND);
IndexWriter iw = new IndexWriter(dir, config);
SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread =
new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0);
controlledRealTimeReopenThread.IsBackground = (true);
controlledRealTimeReopenThread.Start();
IList <ThreadJob> commitThreads = new JCG.List <ThreadJob>();
for (int i = 0; i < 500; i++)
{
if (i > 0 && i % 50 == 0)
{
ThreadJob commitThread = new RunnableAnonymousClass(this, sdp, dir, iw);
commitThread.Start();
commitThreads.Add(commitThread);
}
Document d = new Document();
d.Add(new TextField("count", i + "", Field.Store.NO));
d.Add(new TextField("content", content, Field.Store.YES));
long start = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
long l = tiw.AddDocument(d);
controlledRealTimeReopenThread.WaitForGeneration(l);
long wait = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - start; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
assertTrue("waited too long for generation " + wait, wait < (maxStaleSecs * 1000));
IndexSearcher searcher = sm.Acquire();
TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
sm.Release(searcher);
assertEquals(1, td.TotalHits);
}
foreach (ThreadJob commitThread in commitThreads)
{
commitThread.Join();
}
controlledRealTimeReopenThread.Dispose();
sm.Dispose();
iw.Dispose();
dir.Dispose();
}
// LUCENE-5461
public virtual void TestCRTReopen()
{
//test behaving badly
//should be high enough
int maxStaleSecs = 20;
//build crap data just to store it.
string s = " abcdefghijklmnopqrstuvwxyz ";
char[] chars = s.ToCharArray();
StringBuilder builder = new StringBuilder(2048);
for (int i = 0; i < 2048; i++)
{
builder.Append(chars[Random().Next(chars.Length)]);
}
string content = builder.ToString();
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random()));
config.SetIndexDeletionPolicy(sdp);
config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND);
IndexWriter iw = new IndexWriter(dir, config);
SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0);
controlledRealTimeReopenThread.SetDaemon(true);
controlledRealTimeReopenThread.Start();
IList <Thread> commitThreads = new List <Thread>();
for (int i = 0; i < 500; i++)
{
if (i > 0 && i % 50 == 0)
{
Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw));
commitThread.Start();
commitThreads.Add(commitThread);
}
Document d = new Document();
d.Add(new TextField("count", i + "", Field.Store.NO));
d.Add(new TextField("content", content, Field.Store.YES));
long start = DateTime.Now.Millisecond;
long l = tiw.AddDocument(d);
controlledRealTimeReopenThread.WaitForGeneration(l);
long wait = DateTime.Now.Millisecond - start;
Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait);
IndexSearcher searcher = sm.Acquire();
TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
sm.Release(searcher);
Assert.AreEqual(1, td.TotalHits);
}
foreach (Thread commitThread in commitThreads)
{
commitThread.Join();
}
controlledRealTimeReopenThread.Dispose();
sm.Dispose();
iw.Dispose();
dir.Dispose();
}
public override void Dispose()
{
lock (this)
{
// files that we tried to delete, but couldn't because readers were open.
// all that matters is that we tried! (they will eventually go away)
ISet<string> pendingDeletions = new HashSet<string>(OpenFilesDeleted);
MaybeYield();
if (OpenFiles == null)
{
OpenFiles = new Dictionary<string, int>();
OpenFilesDeleted = new HashSet<string>();
}
if (OpenFiles.Count > 0)
{
// print the first one as its very verbose otherwise
Exception cause = null;
IEnumerator<Exception> stacktraces = OpenFileHandles.Values.GetEnumerator();
if (stacktraces.MoveNext())
{
cause = stacktraces.Current;
}
// RuntimeException instead ofSystem.IO.IOException because
// super() does not throwSystem.IO.IOException currently:
throw new Exception("MockDirectoryWrapper: cannot close: there are still open files: "
+ String.Join(" ,", OpenFiles.ToArray().Select(x => x.Key)), cause);
}
if (OpenLocks.Count > 0)
{
throw new Exception("MockDirectoryWrapper: cannot close: there are still open locks: "
+ String.Join(" ,", OpenLocks.ToArray()));
}
IsOpen = false;
if (CheckIndexOnClose)
{
RandomIOExceptionRate_Renamed = 0.0;
RandomIOExceptionRateOnOpen_Renamed = 0.0;
if (DirectoryReader.IndexExists(this))
{
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("\nNOTE: MockDirectoryWrapper: now crush");
}
Crash(); // corrupt any unsynced-files
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
}
TestUtil.CheckIndex(this, CrossCheckTermVectorsOnClose);
// TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
if (AssertNoUnreferencedFilesOnClose)
{
// now look for unreferenced files: discount ones that we tried to delete but could not
HashSet<string> allFiles = new HashSet<string>(Arrays.AsList(ListAll()));
allFiles.RemoveAll(pendingDeletions);
string[] startFiles = allFiles.ToArray(/*new string[0]*/);
IndexWriterConfig iwc = new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null);
iwc.SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
(new IndexWriter(@in, iwc)).Rollback();
string[] endFiles = @in.ListAll();
ISet<string> startSet = new SortedSet<string>(Arrays.AsList(startFiles));
ISet<string> endSet = new SortedSet<string>(Arrays.AsList(endFiles));
if (pendingDeletions.Contains("segments.gen") && endSet.Contains("segments.gen"))
{
// this is possible if we hit an exception while writing segments.gen, we try to delete it
// and it ends out in pendingDeletions (but IFD wont remove this).
startSet.Add("segments.gen");
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MDW: Unreferenced check: Ignoring segments.gen that we could not delete.");
}
}
// its possible we cannot delete the segments_N on windows if someone has it open and
// maybe other files too, depending on timing. normally someone on windows wouldnt have
// an issue (IFD would nuke this stuff eventually), but we pass NoDeletionPolicy...
foreach (string file in pendingDeletions)
{
if (file.StartsWith("segments") && !file.Equals("segments.gen") && endSet.Contains(file))
{
startSet.Add(file);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MDW: Unreferenced check: Ignoring segments file: " + file + " that we could not delete.");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(@in, file);
}
catch (System.IO.IOException ioe)
{
// OK: likely some of the .si files were deleted
}
try
{
ISet<string> ghosts = new HashSet<string>(sis.Files(@in, false));
foreach (string s in ghosts)
{
if (endSet.Contains(s) && !startSet.Contains(s))
{
Debug.Assert(pendingDeletions.Contains(s));
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MDW: Unreferenced check: Ignoring referenced file: " + s + " " + "from " + file + " that we could not delete.");
}
startSet.Add(s);
}
}
}
catch (Exception t)
{
Console.Error.WriteLine("ERROR processing leftover segments file " + file + ":");
Console.WriteLine(t.ToString());
Console.Write(t.StackTrace);
}
}
}
startFiles = startSet.ToArray(/*new string[0]*/);
endFiles = endSet.ToArray(/*new string[0]*/);
if (!Arrays.Equals(startFiles, endFiles))
{
IList<string> removed = new List<string>();
foreach (string fileName in startFiles)
{
if (!endSet.Contains(fileName))
{
removed.Add(fileName);
}
}
IList<string> added = new List<string>();
foreach (string fileName in endFiles)
{
if (!startSet.Contains(fileName))
{
added.Add(fileName);
}
}
string extras;
if (removed.Count != 0)
{
extras = "\n\nThese files were removed: " + removed;
}
else
{
extras = "";
}
if (added.Count != 0)
{
extras += "\n\nThese files were added (waaaaaaaaaat!): " + added;
}
if (pendingDeletions.Count != 0)
{
extras += "\n\nThese files we had previously tried to delete, but couldn't: " + pendingDeletions;
}
Debug.Assert(false, "unreferenced files: before delete:\n " + Arrays.ToString(startFiles) + "\n after delete:\n " + Arrays.ToString(endFiles) + extras);
}
DirectoryReader ir1 = DirectoryReader.Open(this);
int numDocs1 = ir1.NumDocs;
ir1.Dispose();
(new IndexWriter(this, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null))).Dispose();
DirectoryReader ir2 = DirectoryReader.Open(this);
int numDocs2 = ir2.NumDocs;
ir2.Dispose();
Debug.Assert(numDocs1 == numDocs2, "numDocs changed after opening/closing IW: before=" + numDocs1 + " after=" + numDocs2);
}
}
}
@in.Dispose();
}
}
