public virtual void TestOpenIfChangedNRTToCommit()
{
Directory dir = NewDirectory();
// Can't use RIW because it randomly commits:
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document doc = new Document();
doc.Add(NewStringField("field", "value", Field.Store.NO));
w.AddDocument(doc);
w.Commit();
IList <IndexCommit> commits = DirectoryReader.ListCommits(dir);
Assert.AreEqual(1, commits.Count);
w.AddDocument(doc);
DirectoryReader r = DirectoryReader.Open(w, true);
Assert.AreEqual(2, r.NumDocs);
IndexReader r2 = DirectoryReader.OpenIfChanged(r, commits[0]);
Assert.IsNotNull(r2);
r.Dispose();
Assert.AreEqual(1, r2.NumDocs);
w.Dispose();
r2.Dispose();
dir.Dispose();
}
public override void Run()
{
try
{
DirectoryReader open = null;
for (int i = 0; i < Num; i++)
{
Documents.Document doc = new Documents.Document(); // docs.NextDoc();
doc.Add(NewStringField("id", "test", Field.Store.NO));
Writer.UpdateDocument(new Term("id", "test"), doc);
if (Random().Next(3) == 0)
{
if (open == null)
{
open = DirectoryReader.Open(Writer, true);
}
DirectoryReader reader = DirectoryReader.OpenIfChanged(open);
if (reader != null)
{
open.Dispose();
open = reader;
}
Assert.AreEqual(1, open.NumDocs, "iter: " + i + " numDocs: " + open.NumDocs + " del: " + open.NumDeletedDocs + " max: " + open.MaxDoc);
}
}
if (open != null)
{
open.Dispose();
}
}
catch (Exception e)
{
throw new Exception(e.Message, e);
}
}
public virtual void TestPrepareCommit()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy(5)));
writer.Commit();
for (int i = 0; i < 23; i++)
{
AddDoc(writer);
}
DirectoryReader reader = DirectoryReader.Open(dir);
Assert.AreEqual(0, reader.NumDocs);
writer.PrepareCommit();
IndexReader reader2 = DirectoryReader.Open(dir);
Assert.AreEqual(0, reader2.NumDocs);
writer.Commit();
IndexReader reader3 = DirectoryReader.OpenIfChanged(reader);
Assert.IsNotNull(reader3);
Assert.AreEqual(0, reader.NumDocs);
Assert.AreEqual(0, reader2.NumDocs);
Assert.AreEqual(23, reader3.NumDocs);
reader.Dispose();
reader2.Dispose();
for (int i = 0; i < 17; i++)
{
AddDoc(writer);
}
Assert.AreEqual(23, reader3.NumDocs);
reader3.Dispose();
reader = DirectoryReader.Open(dir);
Assert.AreEqual(23, reader.NumDocs);
reader.Dispose();
writer.PrepareCommit();
reader = DirectoryReader.Open(dir);
Assert.AreEqual(23, reader.NumDocs);
reader.Dispose();
writer.Commit();
reader = DirectoryReader.Open(dir);
Assert.AreEqual(40, reader.NumDocs);
reader.Dispose();
writer.Dispose();
dir.Dispose();
}
internal virtual ReaderCouple RefreshReader(DirectoryReader reader, TestReopen test, int modify, bool hasChanges)
{
UninterruptableMonitor.Enter(createReaderMutex);
try
{
DirectoryReader r = null;
if (test != null)
{
test.ModifyIndex(modify);
r = test.OpenReader();
}
DirectoryReader refreshed = null;
try
{
refreshed = DirectoryReader.OpenIfChanged(reader);
if (refreshed == null)
{
refreshed = reader;
}
}
finally
{
if (refreshed == null && r != null)
{
// Hit exception -- close opened reader
r.Dispose();
}
}
if (hasChanges)
{
if (refreshed == reader)
{
Assert.Fail("No new DirectoryReader instance created during refresh.");
}
}
else
{
if (refreshed != reader)
{
Assert.Fail("New DirectoryReader instance created during refresh even though index had no changes.");
}
}
return(new ReaderCouple(r, refreshed));
}
finally
{
UninterruptableMonitor.Exit(createReaderMutex);
}
}
public override void Run()
{
Random rnd = LuceneTestCase.Random;
while (!stopped)
{
if (index % 2 == 0)
{
// refresh reader synchronized
ReaderCouple c = (outerInstance.RefreshReader(r, test, index, true));
readersToClose.Add(c.newReader);
readersToClose.Add(c.refreshedReader);
readers.Add(c);
// prevent too many readers
break;
}
else
{
// not synchronized
DirectoryReader refreshed = DirectoryReader.OpenIfChanged(r);
if (refreshed == null)
{
refreshed = r;
}
IndexSearcher searcher =
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
outerInstance.
#endif
NewSearcher(refreshed);
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("field1", "a" + rnd.Next(refreshed.MaxDoc))), null, 1000).ScoreDocs;
if (hits.Length > 0)
{
searcher.Doc(hits[0].Doc);
}
if (refreshed != r)
{
refreshed.Dispose();
}
}
UninterruptableMonitor.Enter(this);
try
{
UninterruptableMonitor.Wait(this, TimeSpan.FromMilliseconds(TestUtil.NextInt32(Random, 1, 100)));
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
}
public override void Run()
{
Random rnd = LuceneTestCase.Random();
while (!Stopped)
{
if (Index % 2 == 0)
{
// refresh reader synchronized
ReaderCouple c = (OuterInstance.RefreshReader(r, Test, Index, true));
ReadersToClose.Add(c.NewReader);
ReadersToClose.Add(c.RefreshedReader);
Readers.Add(c);
// prevent too many readers
break;
}
else
{
// not synchronized
DirectoryReader refreshed = DirectoryReader.OpenIfChanged(r);
if (refreshed == null)
{
refreshed = r;
}
IndexSearcher searcher = OuterInstance.NewSearcher(refreshed);
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("field1", "a" + rnd.Next(refreshed.MaxDoc))), null, 1000).ScoreDocs;
if (hits.Length > 0)
{
searcher.Doc(hits[0].Doc);
}
if (refreshed != r)
{
refreshed.Dispose();
}
}
lock (this)
{
Monitor.Wait(this, TimeSpan.FromMilliseconds(TestUtil.NextInt(Random(), 1, 100)));
}
}
}
public override void Run()
{
try
{
Document doc = new Document();
DirectoryReader r = DirectoryReader.Open(dir);
Field f = newStringField("f", "", Field.Store.NO);
doc.Add(f);
int count = 0;
do
{
if (failed)
{
break;
}
for (int j = 0; j < 10; j++)
{
string s = finalI + "_" + Convert.ToString(count++);
f.SetStringValue(s);
w.AddDocument(doc);
w.Commit();
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
Assert.IsNotNull(r2);
Assert.IsTrue(!r2.Equals(r));
r.Dispose();
r = r2;
Assert.AreEqual(1, r.DocFreq(new Term("f", s)), "term=f:" + s + "; r=" + r);
}
} while ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) < endTime); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
r.Dispose();
}
catch (Exception t) when(t.IsThrowable())
{
failed.Value = (true);
throw RuntimeException.Create(t);
}
}
public override void Run()
{
try
{
Document doc = new Document();
DirectoryReader r = DirectoryReader.Open(Dir);
Field f = NewStringField("f", "", Field.Store.NO);
doc.Add(f);
int count = 0;
do
{
if (Failed.Get())
{
break;
}
for (int j = 0; j < 10; j++)
{
string s = FinalI + "_" + Convert.ToString(count++);
f.SetStringValue(s);
w.AddDocument(doc);
w.Commit();
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
Assert.IsNotNull(r2);
Assert.IsTrue(!r2.Equals(r));
r.Dispose();
r = r2;
Assert.AreEqual(1, r.DocFreq(new Term("f", s)), "term=f:" + s + "; r=" + r);
}
} while (Environment.TickCount < EndTime);
r.Dispose();
}
catch (Exception t)
{
Failed.Set(true);
throw new Exception(t.Message, t);
}
}
protected override void DoSearching(TaskScheduler es, long stopTime)
{
bool anyOpenDelFiles = false;
DirectoryReader r = DirectoryReader.Open(m_writer, true);
while (Environment.TickCount < stopTime && !m_failed)
{
if (Random.NextBoolean())
{
if (Verbose)
{
Console.WriteLine("TEST: now reopen r=" + r);
}
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
if (r2 != null)
{
r.Dispose();
r = r2;
}
}
else
{
if (Verbose)
{
Console.WriteLine("TEST: now close reader=" + r);
}
r.Dispose();
m_writer.Commit();
ICollection <string> openDeletedFiles = ((MockDirectoryWrapper)m_dir).GetOpenDeletedFiles();
if (openDeletedFiles.Count > 0)
{
Console.WriteLine("OBD files: " + openDeletedFiles);
}
anyOpenDelFiles |= openDeletedFiles.Count > 0;
//Assert.AreEqual("open but deleted: " + openDeletedFiles, 0, openDeletedFiles.Size());
if (Verbose)
{
Console.WriteLine("TEST: now open");
}
r = DirectoryReader.Open(m_writer, true);
}
if (Verbose)
{
Console.WriteLine("TEST: got new reader=" + r);
}
//System.out.println("numDocs=" + r.NumDocs + "
//openDelFileCount=" + dir.openDeleteFileCount());
if (r.NumDocs > 0)
{
fixedSearcher = new IndexSearcher(r, es);
SmokeTestSearcher(fixedSearcher);
RunSearchThreads(Environment.TickCount + 500);
}
}
r.Dispose();
//System.out.println("numDocs=" + r.NumDocs + " openDelFileCount=" + dir.openDeleteFileCount());
ICollection <string> openDeletedFiles_ = ((MockDirectoryWrapper)m_dir).GetOpenDeletedFiles();
if (openDeletedFiles_.Count > 0)
{
Console.WriteLine("OBD files: " + openDeletedFiles_);
}
anyOpenDelFiles |= openDeletedFiles_.Count > 0;
Assert.IsFalse(anyOpenDelFiles, "saw non-zero open-but-deleted count");
}
public virtual void Test()
{
MockDirectoryWrapper dir = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors"));
dir.PreventDoubleWrite = false;
double rate = Random.NextDouble() * 0.01;
//System.out.println("rate=" + rate);
dir.RandomIOExceptionRateOnOpen = rate;
int iters = AtLeast(20);
LineFileDocs docs = new LineFileDocs(Random, DefaultCodecSupportsDocValues);
IndexReader r = null;
DirectoryReader r2 = null;
bool any = false;
MockDirectoryWrapper dirCopy = null;
int lastNumDocs = 0;
for (int iter = 0; iter < iters; iter++)
{
IndexWriter w = null;
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter);
}
try
{
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
if (VERBOSE)
{
// Do this ourselves instead of relying on LTC so
// we see incrementing messageID:
iwc.SetInfoStream(new TextWriterInfoStream(Console.Out));
}
var ms = iwc.MergeScheduler;
if (ms is IConcurrentMergeScheduler)
{
((IConcurrentMergeScheduler)ms).SetSuppressExceptions();
}
w = new IndexWriter(dir, iwc);
if (r != null && Random.Next(5) == 3)
{
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: addIndexes IR[]");
}
w.AddIndexes(new IndexReader[] { r });
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: addIndexes Directory[]");
}
w.AddIndexes(new Directory[] { dirCopy });
}
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: addDocument");
}
w.AddDocument(docs.NextDoc());
}
dir.RandomIOExceptionRateOnOpen = 0.0;
w.Dispose();
w = null;
// NOTE: this is O(N^2)! Only enable for temporary debugging:
//dir.setRandomIOExceptionRateOnOpen(0.0);
//TestUtil.CheckIndex(dir);
//dir.setRandomIOExceptionRateOnOpen(rate);
// Verify numDocs only increases, to catch IndexWriter
// accidentally deleting the index:
dir.RandomIOExceptionRateOnOpen = 0.0;
Assert.IsTrue(DirectoryReader.IndexExists(dir));
if (r2 == null)
{
r2 = DirectoryReader.Open(dir);
}
else
{
DirectoryReader r3 = DirectoryReader.OpenIfChanged(r2);
if (r3 != null)
{
r2.Dispose();
r2 = r3;
}
}
Assert.IsTrue(r2.NumDocs >= lastNumDocs, "before=" + lastNumDocs + " after=" + r2.NumDocs);
lastNumDocs = r2.NumDocs;
//System.out.println("numDocs=" + lastNumDocs);
dir.RandomIOExceptionRateOnOpen = rate;
any = true;
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + ": success");
}
}
catch (IOException ioe)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + ": exception");
Console.WriteLine(ioe.ToString());
Console.Write(ioe.StackTrace);
}
if (w != null)
{
// NOTE: leave random IO exceptions enabled here,
// to verify that rollback does not try to write
// anything:
w.Rollback();
}
}
if (any && r == null && Random.NextBoolean())
{
// Make a copy of a non-empty index so we can use
// it to addIndexes later:
dir.RandomIOExceptionRateOnOpen = 0.0;
r = DirectoryReader.Open(dir);
dirCopy = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors.copy"));
HashSet <string> files = new HashSet <string>();
foreach (string file in dir.ListAll())
{
dir.Copy(dirCopy, file, file, IOContext.DEFAULT);
files.Add(file);
}
dirCopy.Sync(files);
// Have IW kiss the dir so we remove any leftover
// files ... we can easily have leftover files at
// the time we take a copy because we are holding
// open a reader:
(new IndexWriter(dirCopy, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)))).Dispose();
dirCopy.RandomIOExceptionRate = rate;
dir.RandomIOExceptionRateOnOpen = rate;
}
}
if (r2 != null)
{
r2.Dispose();
}
if (r != null)
{
r.Dispose();
dirCopy.Dispose();
}
dir.Dispose();
}
public override void Run()
{
try
{
while (Operations.Get() > 0)
{
int oper = rand.Next(100);
if (oper < CommitPercent)
{
if (NumCommitting.IncrementAndGet() <= MaxConcurrentCommits)
{
IDictionary <int, long> newCommittedModel;
long version;
DirectoryReader oldReader;
lock (OuterInstance)
{
newCommittedModel = new Dictionary <int, long>(OuterInstance.Model); // take a snapshot
version = OuterInstance.SnapshotCount++;
oldReader = OuterInstance.Reader;
oldReader.IncRef(); // increment the reference since we will use this for reopening
}
DirectoryReader newReader;
if (rand.Next(100) < SoftCommitPercent)
{
// assertU(h.Commit("softCommit","true"));
if (Random().NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": call writer.getReader");
}
newReader = Writer.GetReader(true);
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": reopen reader=" + oldReader + " version=" + version);
}
newReader = DirectoryReader.OpenIfChanged(oldReader, Writer.w, true);
}
}
else
{
// assertU(commit());
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": commit+reopen reader=" + oldReader + " version=" + version);
}
Writer.Commit();
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": now reopen after commit");
}
newReader = DirectoryReader.OpenIfChanged(oldReader);
}
// Code below assumes newReader comes w/
// extra ref:
if (newReader == null)
{
oldReader.IncRef();
newReader = oldReader;
}
oldReader.DecRef();
lock (OuterInstance)
{
// install the new reader if it's newest (and check the current version since another reader may have already been installed)
//System.out.println(Thread.currentThread().getName() + ": newVersion=" + newReader.getVersion());
Debug.Assert(newReader.RefCount > 0);
Debug.Assert(OuterInstance.Reader.RefCount > 0);
if (newReader.Version > OuterInstance.Reader.Version)
{
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": install new reader=" + newReader);
}
OuterInstance.Reader.DecRef();
OuterInstance.Reader = newReader;
// Silly: forces fieldInfos to be
// loaded so we don't hit IOE on later
// reader.toString
newReader.ToString();
// install this snapshot only if it's newer than the current one
if (version >= OuterInstance.CommittedModelClock)
{
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": install new model version=" + version);
}
OuterInstance.CommittedModel = newCommittedModel;
OuterInstance.CommittedModelClock = version;
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": skip install new model version=" + version);
}
}
}
else
{
// if the same reader, don't decRef.
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": skip install new reader=" + newReader);
}
newReader.DecRef();
}
}
}
NumCommitting.DecrementAndGet();
}
else
{
int id = rand.Next(Ndocs);
object sync = OuterInstance.SyncArr[id];
// set the lastId before we actually change it sometimes to try and
// uncover more race conditions between writing and reading
bool before = Random().NextBoolean();
if (before)
{
OuterInstance.LastId = id;
}
// We can't concurrently update the same document and retain our invariants of increasing values
// since we can't guarantee what order the updates will be executed.
lock (sync)
{
long val = OuterInstance.Model[id];
long nextVal = Math.Abs(val) + 1;
if (oper < CommitPercent + DeletePercent)
{
// assertU("<delete><id>" + id + "</id></delete>");
// add tombstone first
if (Tombstones)
{
Document d = new Document();
d.Add(OuterInstance.NewStringField("id", "-" + Convert.ToString(id), Documents.Field.Store.YES));
d.Add(OuterInstance.NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType));
Writer.UpdateDocument(new Term("id", "-" + Convert.ToString(id)), d);
}
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": term delDocs id:" + id + " nextVal=" + nextVal);
}
Writer.DeleteDocuments(new Term("id", Convert.ToString(id)));
OuterInstance.Model[id] = -nextVal;
}
else if (oper < CommitPercent + DeletePercent + DeleteByQueryPercent)
{
//assertU("<delete><query>id:" + id + "</query></delete>");
// add tombstone first
if (Tombstones)
{
Document d = new Document();
d.Add(OuterInstance.NewStringField("id", "-" + Convert.ToString(id), Documents.Field.Store.YES));
d.Add(OuterInstance.NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType));
Writer.UpdateDocument(new Term("id", "-" + Convert.ToString(id)), d);
}
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": query delDocs id:" + id + " nextVal=" + nextVal);
}
Writer.DeleteDocuments(new TermQuery(new Term("id", Convert.ToString(id))));
OuterInstance.Model[id] = -nextVal;
}
else
{
// assertU(adoc("id",Integer.toString(id), field, Long.toString(nextVal)));
Document d = new Document();
d.Add(OuterInstance.NewStringField("id", Convert.ToString(id), Documents.Field.Store.YES));
d.Add(OuterInstance.NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType));
if (VERBOSE)
{
Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": u id:" + id + " val=" + nextVal);
}
Writer.UpdateDocument(new Term("id", Convert.ToString(id)), d);
if (Tombstones)
{
// remove tombstone after new addition (this should be optional?)
Writer.DeleteDocuments(new Term("id", "-" + Convert.ToString(id)));
}
OuterInstance.Model[id] = nextVal;
}
}
if (!before)
{
OuterInstance.LastId = id;
}
}
}
}
catch (Exception e)
{
Console.WriteLine(Thread.CurrentThread.Name + ": FAILED: unexpected exception");
Console.WriteLine(e.StackTrace);
throw new Exception(e.Message, e);
}
}
public virtual void TestIndexing()
{
DirectoryInfo tmpDir = CreateTempDir("TestNeverDelete");
BaseDirectoryWrapper d = NewFSDirectory(tmpDir);
// We want to "see" files removed if Lucene removed
// them. this is still worth running on Windows since
// some files the IR opens and closes.
if (d is MockDirectoryWrapper)
{
((MockDirectoryWrapper)d).NoDeleteOpenFile = false;
}
RandomIndexWriter w = new RandomIndexWriter(Random(), d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
w.w.Config.SetMaxBufferedDocs(TestUtil.NextInt(Random(), 5, 30));
w.Commit();
ThreadClass[] indexThreads = new ThreadClass[Random().Next(4)];
long stopTime = Environment.TickCount + AtLeast(1000);
for (int x = 0; x < indexThreads.Length; x++)
{
indexThreads[x] = new ThreadAnonymousInnerClassHelper(w, stopTime, NewStringField, NewTextField);
indexThreads[x].Name = "Thread " + x;
indexThreads[x].Start();
}
HashSet <string> allFiles = new HashSet <string>();
DirectoryReader r = DirectoryReader.Open(d);
while (Environment.TickCount < stopTime)
{
IndexCommit ic = r.IndexCommit;
if (VERBOSE)
{
Console.WriteLine("TEST: check files: " + ic.FileNames);
}
allFiles.AddAll(ic.FileNames);
// Make sure no old files were removed
foreach (string fileName in allFiles)
{
Assert.IsTrue(SlowFileExists(d, fileName), "file " + fileName + " does not exist");
}
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
if (r2 != null)
{
r.Dispose();
r = r2;
}
Thread.Sleep(1);
}
r.Dispose();
foreach (ThreadClass t in indexThreads)
{
t.Join();
}
w.Dispose();
d.Dispose();
System.IO.Directory.Delete(tmpDir.FullName, true);
}
public virtual void TestReopenOnCommit()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetIndexDeletionPolicy(new KeepAllCommits <IndexCommit>()).SetMaxBufferedDocs(-1).SetMergePolicy(NewLogMergePolicy(10)));
for (int i = 0; i < 4; i++)
{
Document doc = new Document();
doc.Add(NewStringField("id", "" + i, Field.Store.NO));
writer.AddDocument(doc);
IDictionary <string, string> data = new Dictionary <string, string>();
data["index"] = i + "";
writer.CommitData = data;
writer.Commit();
}
for (int i = 0; i < 4; i++)
{
writer.DeleteDocuments(new Term("id", "" + i));
IDictionary <string, string> data = new Dictionary <string, string>();
data["index"] = (4 + i) + "";
writer.CommitData = data;
writer.Commit();
}
writer.Dispose();
DirectoryReader r = DirectoryReader.Open(dir);
Assert.AreEqual(0, r.NumDocs);
ICollection <IndexCommit> commits = DirectoryReader.ListCommits(dir);
foreach (IndexCommit commit in commits)
{
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r, commit);
Assert.IsNotNull(r2);
Assert.IsTrue(r2 != r);
IDictionary <string, string> s = commit.UserData;
int v;
if (s.Count == 0)
{
// First commit created by IW
v = -1;
}
else
{
v = Convert.ToInt32(s["index"]);
}
if (v < 4)
{
Assert.AreEqual(1 + v, r2.NumDocs);
}
else
{
Assert.AreEqual(7 - v, r2.NumDocs);
}
r.Dispose();
r = r2;
}
r.Dispose();
dir.Dispose();
}
protected override void DoSearching(TaskScheduler es, DateTime stopTime)
{
bool anyOpenDelFiles = false;
DirectoryReader r = DirectoryReader.Open(Writer, true);
while (DateTime.UtcNow < stopTime && !Failed.Get())
{
if (Random().NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: now reopen r=" + r);
}
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
if (r2 != null)
{
r.Dispose();
r = r2;
}
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: now close reader=" + r);
}
r.Dispose();
Writer.Commit();
ISet <string> openDeletedFiles = ((MockDirectoryWrapper)Dir).OpenDeletedFiles;
if (openDeletedFiles.Count > 0)
{
Console.WriteLine("OBD files: " + openDeletedFiles);
}
anyOpenDelFiles |= openDeletedFiles.Count > 0;
//Assert.AreEqual("open but deleted: " + openDeletedFiles, 0, openDeletedFiles.Size());
if (VERBOSE)
{
Console.WriteLine("TEST: now open");
}
r = DirectoryReader.Open(Writer, true);
}
if (VERBOSE)
{
Console.WriteLine("TEST: got new reader=" + r);
}
//System.out.println("numDocs=" + r.NumDocs + "
//openDelFileCount=" + dir.openDeleteFileCount());
if (r.NumDocs > 0)
{
FixedSearcher = new IndexSearcher(r, es);
SmokeTestSearcher(FixedSearcher);
RunSearchThreads(DateTime.UtcNow.AddMilliseconds(500));
}
}
r.Dispose();
//System.out.println("numDocs=" + r.NumDocs + " openDelFileCount=" + dir.openDeleteFileCount());
ISet <string> openDeletedFiles_ = ((MockDirectoryWrapper)Dir).OpenDeletedFiles;
if (openDeletedFiles_.Count > 0)
{
Console.WriteLine("OBD files: " + openDeletedFiles_);
}
anyOpenDelFiles |= openDeletedFiles_.Count > 0;
Assert.IsFalse(anyOpenDelFiles, "saw non-zero open-but-deleted count");
}
public virtual void TestThreadSafety()
{
Directory dir = NewDirectory();
// NOTE: this also controls the number of threads!
int n = TestUtil.NextInt(Random(), 20, 40);
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
for (int i = 0; i < n; i++)
{
writer.AddDocument(CreateDocument(i, 3));
}
writer.ForceMerge(1);
writer.Dispose();
TestReopen test = new TestReopenAnonymousInnerClassHelper3(this, dir, n);
IList <ReaderCouple> readers = new SynchronizedCollection <ReaderCouple>();
DirectoryReader firstReader = DirectoryReader.Open(dir);
DirectoryReader reader = firstReader;
ReaderThread[] threads = new ReaderThread[n];
ISet <DirectoryReader> readersToClose = new ConcurrentHashSet <DirectoryReader>(new HashSet <DirectoryReader>());
for (int i = 0; i < n; i++)
{
if (i % 2 == 0)
{
DirectoryReader refreshed = DirectoryReader.OpenIfChanged(reader);
if (refreshed != null)
{
readersToClose.Add(reader);
reader = refreshed;
}
}
DirectoryReader r = reader;
int index = i;
ReaderThreadTask task;
if (i < 4 || (i >= 10 && i < 14) || i > 18)
{
task = new ReaderThreadTaskAnonymousInnerClassHelper(this, test, readers, readersToClose, r, index);
}
else
{
task = new ReaderThreadTaskAnonymousInnerClassHelper2(this, readers);
}
threads[i] = new ReaderThread(task);
threads[i].Start();
}
lock (this)
{
Monitor.Wait(this, TimeSpan.FromMilliseconds(1000));
}
for (int i = 0; i < n; i++)
{
if (threads[i] != null)
{
threads[i].StopThread();
}
}
for (int i = 0; i < n; i++)
{
if (threads[i] != null)
{
threads[i].Join();
if (threads[i].Error != null)
{
string msg = "Error occurred in thread " + threads[i].Name + ":\n" + threads[i].Error.Message;
Assert.Fail(msg);
}
}
}
foreach (DirectoryReader readerToClose in readersToClose)
{
readerToClose.Dispose();
}
firstReader.Dispose();
reader.Dispose();
foreach (DirectoryReader readerToClose in readersToClose)
{
AssertReaderClosed(readerToClose, true);
}
AssertReaderClosed(reader, true);
AssertReaderClosed(firstReader, true);
dir.Dispose();
}
private void DoTestReopenWithCommit(Random random, Directory dir, bool withReopen)
{
IndexWriter iwriter = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetOpenMode(OpenMode_e.CREATE).SetMergeScheduler(new SerialMergeScheduler()).SetMergePolicy(NewLogMergePolicy()));
iwriter.Commit();
DirectoryReader reader = DirectoryReader.Open(dir);
try
{
int M = 3;
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.Tokenized = false;
FieldType customType2 = new FieldType(TextField.TYPE_STORED);
customType2.Tokenized = false;
customType2.OmitNorms = true;
FieldType customType3 = new FieldType();
customType3.Stored = true;
for (int i = 0; i < 4; i++)
{
for (int j = 0; j < M; j++)
{
Document doc = new Document();
doc.Add(NewField("id", i + "_" + j, customType));
doc.Add(NewField("id2", i + "_" + j, customType2));
doc.Add(NewField("id3", i + "_" + j, customType3));
iwriter.AddDocument(doc);
if (i > 0)
{
int k = i - 1;
int n = j + k * M;
Document prevItereationDoc = reader.Document(n);
Assert.IsNotNull(prevItereationDoc);
string id = prevItereationDoc.Get("id");
Assert.AreEqual(k + "_" + j, id);
}
}
iwriter.Commit();
if (withReopen)
{
// reopen
DirectoryReader r2 = DirectoryReader.OpenIfChanged(reader);
if (r2 != null)
{
reader.Dispose();
reader = r2;
}
}
else
{
// recreate
reader.Dispose();
reader = DirectoryReader.Open(dir);
}
}
}
finally
{
iwriter.Dispose();
reader.Dispose();
}
}
public override void Run()
{
DirectoryReader currentReader = null;
Random random = LuceneTestCase.Random();
try
{
Document doc = new Document();
doc.Add(new TextField("id", "1", Field.Store.NO));
Writer.AddDocument(doc);
Holder.Reader = currentReader = Writer.GetReader(true);
Term term = new Term("id");
for (int i = 0; i < NumOps && !Holder.Stop; i++)
{
float nextOp = (float)random.NextDouble();
if (nextOp < 0.3)
{
term.Set("id", new BytesRef("1"));
Writer.UpdateDocument(term, doc);
}
else if (nextOp < 0.5)
{
Writer.AddDocument(doc);
}
else
{
term.Set("id", new BytesRef("1"));
Writer.DeleteDocuments(term);
}
if (Holder.Reader != currentReader)
{
Holder.Reader = currentReader;
if (Countdown)
{
Countdown = false;
Latch.Signal();
}
}
if (random.NextBoolean())
{
Writer.Commit();
DirectoryReader newReader = DirectoryReader.OpenIfChanged(currentReader);
if (newReader != null)
{
currentReader.DecRef();
currentReader = newReader;
}
if (currentReader.NumDocs == 0)
{
Writer.AddDocument(doc);
}
}
}
}
catch (Exception e)
{
Failed = e;
}
finally
{
Holder.Reader = null;
if (Countdown)
{
Latch.Signal();
}
if (currentReader != null)
{
try
{
currentReader.DecRef();
}
catch (IOException e)
{
}
}
}
if (VERBOSE)
{
Console.WriteLine("writer stopped - forced by reader: " + Holder.Stop);
}
}
public override void Run()
{
DirectoryReader reader = null;
bool success = false;
try
{
Random random = Random;
while (numUpdates.GetAndDecrement() > 0)
{
double group = random.NextDouble();
Term t;
if (group < 0.1)
{
t = new Term("updKey", "g0");
}
else if (group < 0.5)
{
t = new Term("updKey", "g1");
}
else if (group < 0.8)
{
t = new Term("updKey", "g2");
}
else
{
t = new Term("updKey", "g3");
}
// System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
if (random.NextBoolean()) // sometimes unset a value
{
// System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=UNSET");
writer.UpdateBinaryDocValue(t, f, null);
writer.UpdateNumericDocValue(t, cf, null);
}
else
{
long updValue = random.Next();
// System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=" + updValue);
writer.UpdateBinaryDocValue(t, f, TestBinaryDocValuesUpdates.ToBytes(updValue));
writer.UpdateNumericDocValue(t, cf, updValue * 2);
}
if (random.NextDouble() < 0.2)
{
// delete a random document
int doc = random.Next(numDocs);
// System.out.println("[" + Thread.currentThread().getName() + "] deleteDoc=doc" + doc);
writer.DeleteDocuments(new Term("id", "doc" + doc));
}
if (random.NextDouble() < 0.05) // commit every 20 updates on average
{
// System.out.println("[" + Thread.currentThread().getName() + "] commit");
writer.Commit();
}
if (random.NextDouble() < 0.1) // reopen NRT reader (apply updates), on average once every 10 updates
{
if (reader is null)
{
// System.out.println("[" + Thread.currentThread().getName() + "] open NRT");
reader = DirectoryReader.Open(writer, true);
}
else
{
// System.out.println("[" + Thread.currentThread().getName() + "] reopen NRT");
DirectoryReader r2 = DirectoryReader.OpenIfChanged(reader, writer, true);
if (r2 != null)
{
reader.Dispose();
reader = r2;
}
}
}
}
// System.out.println("[" + Thread.currentThread().getName() + "] DONE");
success = true;
}
catch (Exception e) when(e.IsIOException())
{
throw RuntimeException.Create(e);
}
finally
{
if (reader != null)
{
try
{
reader.Dispose();
}
catch (Exception e) when(e.IsIOException())
{
if (success) // suppress this exception only if there was another exception
{
throw RuntimeException.Create(e);
}
}
}
done.Signal();
}
}
public override void Run()
{
DirectoryReader currentReader = null;
Random random = LuceneTestCase.Random;
try
{
Document doc = new Document();
doc.Add(new TextField("id", "1", Field.Store.NO));
writer.AddDocument(doc);
holder.reader = currentReader = writer.GetReader(true);
Term term = new Term("id");
for (int i = 0; i < numOps && !holder.stop; i++)
{
float nextOp = (float)random.NextDouble();
if (nextOp < 0.3)
{
term.Set("id", new BytesRef("1"));
writer.UpdateDocument(term, doc);
}
else if (nextOp < 0.5)
{
writer.AddDocument(doc);
}
else
{
term.Set("id", new BytesRef("1"));
writer.DeleteDocuments(term);
}
if (holder.reader != currentReader)
{
holder.reader = currentReader;
if (countdown)
{
countdown = false;
latch.Signal();
}
}
if (random.NextBoolean())
{
writer.Commit();
DirectoryReader newReader = DirectoryReader.OpenIfChanged(currentReader);
if (newReader != null)
{
currentReader.DecRef();
currentReader = newReader;
}
if (currentReader.NumDocs == 0)
{
writer.AddDocument(doc);
}
}
}
}
catch (Exception e)
{
failed = e;
}
finally
{
holder.reader = null;
if (countdown)
{
latch.Signal();
}
if (currentReader != null)
{
try
{
currentReader.DecRef();
}
#pragma warning disable 168
catch (IOException e)
#pragma warning restore 168
{
}
}
}
if (Verbose)
{
Console.WriteLine("writer stopped - forced by reader: " + holder.stop);
}
}
public virtual void TestIndexing()
{
DirectoryInfo tmpDir = CreateTempDir("TestNeverDelete");
BaseDirectoryWrapper d = NewFSDirectory(tmpDir);
// We want to "see" files removed if Lucene removed
// them. this is still worth running on Windows since
// some files the IR opens and closes.
if (d is MockDirectoryWrapper)
{
((MockDirectoryWrapper)d).NoDeleteOpenFile = false;
}
RandomIndexWriter w = new RandomIndexWriter(Random, d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
w.IndexWriter.Config.SetMaxBufferedDocs(TestUtil.NextInt32(Random, 5, 30));
w.Commit();
ThreadJob[] indexThreads = new ThreadJob[Random.Next(4)];
long stopTime = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) + AtLeast(1000); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
for (int x = 0; x < indexThreads.Length; x++)
{
indexThreads[x] = new ThreadAnonymousClass(w, stopTime, NewStringField, NewTextField);
indexThreads[x].Name = "Thread " + x;
indexThreads[x].Start();
}
ISet <string> allFiles = new JCG.HashSet <string>();
DirectoryReader r = DirectoryReader.Open(d);
while (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond < stopTime) // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
{
IndexCommit ic = r.IndexCommit;
if (Verbose)
{
Console.WriteLine("TEST: check files: " + ic.FileNames);
}
allFiles.UnionWith(ic.FileNames);
// Make sure no old files were removed
foreach (string fileName in allFiles)
{
Assert.IsTrue(SlowFileExists(d, fileName), "file " + fileName + " does not exist");
}
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
if (r2 != null)
{
r.Dispose();
r = r2;
}
Thread.Sleep(1);
}
r.Dispose();
foreach (ThreadJob t in indexThreads)
{
t.Join();
}
w.Dispose();
d.Dispose();
System.IO.Directory.Delete(tmpDir.FullName, true);
}
public virtual void TestManyReopensAndFields()
{
Directory dir = NewDirectory();
Random random = Random;
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
LogMergePolicy lmp = NewLogMergePolicy();
lmp.MergeFactor = 3; // merge often
conf.SetMergePolicy(lmp);
IndexWriter writer = new IndexWriter(dir, conf);
bool isNRT = random.NextBoolean();
DirectoryReader reader;
if (isNRT)
{
reader = DirectoryReader.Open(writer, true);
}
else
{
writer.Commit();
reader = DirectoryReader.Open(dir);
}
int numFields = random.Next(4) + 3; // 3-7
int numNDVFields = random.Next(numFields / 2) + 1; // 1-3
long[] fieldValues = new long[numFields];
bool[] fieldHasValue = new bool[numFields];
Arrays.Fill(fieldHasValue, true);
for (int i = 0; i < fieldValues.Length; i++)
{
fieldValues[i] = 1;
}
int numRounds = AtLeast(15);
int docID = 0;
for (int i = 0; i < numRounds; i++)
{
int numDocs = AtLeast(5);
// System.out.println("[" + Thread.currentThread().getName() + "]: round=" + i + ", numDocs=" + numDocs);
for (int j = 0; j < numDocs; j++)
{
Document doc = new Document();
doc.Add(new StringField("id", "doc-" + docID, Store.NO));
doc.Add(new StringField("key", "all", Store.NO)); // update key
// add all fields with their current value
for (int f = 0; f < fieldValues.Length; f++)
{
if (f < numNDVFields)
{
doc.Add(new NumericDocValuesField("f" + f, fieldValues[f]));
}
else
{
doc.Add(new BinaryDocValuesField("f" + f, TestBinaryDocValuesUpdates.ToBytes(fieldValues[f])));
}
}
writer.AddDocument(doc);
++docID;
}
// if field's value was unset before, unset it from all new added documents too
for (int field = 0; field < fieldHasValue.Length; field++)
{
if (!fieldHasValue[field])
{
if (field < numNDVFields)
{
writer.UpdateNumericDocValue(new Term("key", "all"), "f" + field, null);
}
else
{
writer.UpdateBinaryDocValue(new Term("key", "all"), "f" + field, null);
}
}
}
int fieldIdx = random.Next(fieldValues.Length);
string updateField = "f" + fieldIdx;
if (random.NextBoolean())
{
// System.out.println("[" + Thread.currentThread().getName() + "]: unset field '" + updateField + "'");
fieldHasValue[fieldIdx] = false;
if (fieldIdx < numNDVFields)
{
writer.UpdateNumericDocValue(new Term("key", "all"), updateField, null);
}
else
{
writer.UpdateBinaryDocValue(new Term("key", "all"), updateField, null);
}
}
else
{
fieldHasValue[fieldIdx] = true;
if (fieldIdx < numNDVFields)
{
writer.UpdateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
}
else
{
writer.UpdateBinaryDocValue(new Term("key", "all"), updateField, TestBinaryDocValuesUpdates.ToBytes(++fieldValues[fieldIdx]));
}
// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
}
if (random.NextDouble() < 0.2)
{
int deleteDoc = random.Next(docID); // might also delete an already deleted document, ok!
writer.DeleteDocuments(new Term("id", "doc-" + deleteDoc));
// System.out.println("[" + Thread.currentThread().getName() + "]: deleted document: doc-" + deleteDoc);
}
// verify reader
if (!isNRT)
{
writer.Commit();
}
// System.out.println("[" + Thread.currentThread().getName() + "]: reopen reader: " + reader);
DirectoryReader newReader = DirectoryReader.OpenIfChanged(reader);
Assert.IsNotNull(newReader);
reader.Dispose();
reader = newReader;
// System.out.println("[" + Thread.currentThread().getName() + "]: reopened reader: " + reader);
Assert.IsTrue(reader.NumDocs > 0); // we delete at most one document per round
BytesRef scratch = new BytesRef();
foreach (AtomicReaderContext context in reader.Leaves)
{
AtomicReader r = context.AtomicReader;
// System.out.println(((SegmentReader) r).getSegmentName());
IBits liveDocs = r.LiveDocs;
for (int field = 0; field < fieldValues.Length; field++)
{
string f = "f" + field;
BinaryDocValues bdv = r.GetBinaryDocValues(f);
NumericDocValues ndv = r.GetNumericDocValues(f);
IBits docsWithField = r.GetDocsWithField(f);
if (field < numNDVFields)
{
Assert.IsNotNull(ndv);
Assert.IsNull(bdv);
}
else
{
Assert.IsNull(ndv);
Assert.IsNotNull(bdv);
}
int maxDoc = r.MaxDoc;
for (int doc = 0; doc < maxDoc; doc++)
{
if (liveDocs is null || liveDocs.Get(doc))
{
// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch));
if (fieldHasValue[field])
{
Assert.IsTrue(docsWithField.Get(doc));
if (field < numNDVFields)
{
Assert.AreEqual(fieldValues[field], ndv.Get(doc), "invalid value for doc=" + doc + ", field=" + f + ", reader=" + r);
}
else
{
Assert.AreEqual(fieldValues[field], TestBinaryDocValuesUpdates.GetValue(bdv, doc, scratch), "invalid value for doc=" + doc + ", field=" + f + ", reader=" + r);
}
}
else
{
Assert.IsFalse(docsWithField.Get(doc));
}
}
}
}
}
// System.out.println();
}
IOUtils.Dispose(writer, reader, dir);
}
public virtual void TestPrepareCommitRollback()
{
Directory dir = NewDirectory();
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).PreventDoubleWrite = false;
}
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy(5)));
writer.Commit();
for (int i = 0; i < 23; i++)
{
AddDoc(writer);
}
DirectoryReader reader = DirectoryReader.Open(dir);
Assert.AreEqual(0, reader.NumDocs);
writer.PrepareCommit();
IndexReader reader2 = DirectoryReader.Open(dir);
Assert.AreEqual(0, reader2.NumDocs);
writer.Rollback();
IndexReader reader3 = DirectoryReader.OpenIfChanged(reader);
Assert.IsNull(reader3);
Assert.AreEqual(0, reader.NumDocs);
Assert.AreEqual(0, reader2.NumDocs);
reader.Dispose();
reader2.Dispose();
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
for (int i = 0; i < 17; i++)
{
AddDoc(writer);
}
reader = DirectoryReader.Open(dir);
Assert.AreEqual(0, reader.NumDocs);
reader.Dispose();
writer.PrepareCommit();
reader = DirectoryReader.Open(dir);
Assert.AreEqual(0, reader.NumDocs);
reader.Dispose();
writer.Commit();
reader = DirectoryReader.Open(dir);
Assert.AreEqual(17, reader.NumDocs);
reader.Dispose();
writer.Dispose();
dir.Dispose();
}