private void SetUpDirs(Directory dir, Directory aux)
{
IndexWriter writer = null;
writer = NewWriter(dir, true);
writer.SetMaxBufferedDocs(1000);
// add 1000 documents in 1 segment
AddDocs(writer, 1000);
Assert.AreEqual(1000, writer.DocCount());
Assert.AreEqual(1, writer.GetSegmentCount());
writer.Close();
writer = NewWriter(aux, true);
writer.SetUseCompoundFile(false); // use one without a compound file
writer.SetMaxBufferedDocs(100);
writer.SetMergeFactor(10);
// add 30 documents in 3 segments
for (int i = 0; i < 3; i++)
{
AddDocs(writer, 10);
writer.Close();
writer = NewWriter(aux, false);
writer.SetUseCompoundFile(false); // use one without a compound file
writer.SetMaxBufferedDocs(100);
writer.SetMergeFactor(10);
}
Assert.AreEqual(30, writer.DocCount());
Assert.AreEqual(3, writer.GetSegmentCount());
writer.Close();
}
public virtual void TestNoOverMerge()
{
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
writer.SetMergeFactor(10);
writer.SetMergePolicy(new LogDocMergePolicy(writer));
bool noOverMerge = false;
for (int i = 0; i < 100; i++)
{
AddDoc(writer);
CheckInvariants(writer);
if (writer.GetNumBufferedDocuments() + writer.GetSegmentCount() >= 18)
{
noOverMerge = true;
}
}
Assert.IsTrue(noOverMerge);
writer.Close();
}
private void CheckInvariants(IndexWriter writer)
{
_TestUtil.SyncConcurrentMerges(writer);
int maxBufferedDocs = writer.GetMaxBufferedDocs();
int mergeFactor = writer.GetMergeFactor();
int maxMergeDocs = writer.GetMaxMergeDocs();
int ramSegmentCount = writer.GetNumBufferedDocuments();
Assert.IsTrue(ramSegmentCount < maxBufferedDocs);
int lowerBound = -1;
int upperBound = maxBufferedDocs;
int numSegments = 0;
int segmentCount = writer.GetSegmentCount();
for (int i = segmentCount - 1; i >= 0; i--)
{
int docCount = writer.GetDocCount(i);
Assert.IsTrue(docCount > lowerBound);
if (docCount <= upperBound)
{
numSegments++;
}
else
{
if (upperBound * mergeFactor <= maxMergeDocs)
{
Assert.IsTrue(numSegments < mergeFactor);
}
do
{
lowerBound = upperBound;
upperBound *= mergeFactor;
}while (docCount > upperBound);
numSegments = 1;
}
}
if (upperBound * mergeFactor <= maxMergeDocs)
{
Assert.IsTrue(numSegments < mergeFactor);
}
System.String[] files = writer.GetDirectory().ListAll();
int segmentCfsCount = 0;
for (int i = 0; i < files.Length; i++)
{
if (files[i].EndsWith(".cfs"))
{
segmentCfsCount++;
}
}
Assert.AreEqual(segmentCount, segmentCfsCount);
}
public virtual void TestRAMDeletes()
{
for (int pass = 0; pass < 2; pass++)
{
for (int t = 0; t < 2; t++)
{
bool autoCommit = (0 == pass);
Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
modifier.SetMaxBufferedDocs(4);
modifier.SetMaxBufferedDeleteTerms(4);
int id = 0;
int value_Renamed = 100;
AddDoc(modifier, ++id, value_Renamed);
if (0 == t)
{
modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
}
else
{
modifier.DeleteDocuments(new TermQuery(new Term("value", System.Convert.ToString(value_Renamed))));
}
AddDoc(modifier, ++id, value_Renamed);
if (0 == t)
{
modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
Assert.AreEqual(2, modifier.GetNumBufferedDeleteTerms());
Assert.AreEqual(1, modifier.GetBufferedDeleteTermsSize());
}
else
{
modifier.DeleteDocuments(new TermQuery(new Term("value", System.Convert.ToString(value_Renamed))));
}
AddDoc(modifier, ++id, value_Renamed);
Assert.AreEqual(0, modifier.GetSegmentCount());
modifier.Flush();
modifier.Commit();
IndexReader reader = IndexReader.Open(dir);
Assert.AreEqual(1, reader.NumDocs());
int hitCount = GetHitCount(dir, new Term("id", System.Convert.ToString(id)));
Assert.AreEqual(1, hitCount);
reader.Close();
modifier.Close();
dir.Close();
}
}
}
public virtual void TestMoreMerges()
{
// main directory
Directory dir = new RAMDirectory();
// auxiliary directory
Directory aux = new RAMDirectory();
Directory aux2 = new RAMDirectory();
SetUpDirs(dir, aux);
IndexWriter writer = NewWriter(aux2, true);
writer.SetMaxBufferedDocs(100);
writer.SetMergeFactor(10);
writer.AddIndexesNoOptimize(new Directory[] { aux });
Assert.AreEqual(30, writer.DocCount());
Assert.AreEqual(3, writer.GetSegmentCount());
writer.Close();
IndexReader reader = IndexReader.Open(aux);
for (int i = 0; i < 27; i++)
{
reader.DeleteDocument(i);
}
Assert.AreEqual(3, reader.NumDocs());
reader.Close();
reader = IndexReader.Open(aux2);
for (int i = 0; i < 8; i++)
{
reader.DeleteDocument(i);
}
Assert.AreEqual(22, reader.NumDocs());
reader.Close();
writer = NewWriter(dir, false);
writer.SetMaxBufferedDocs(6);
writer.SetMergeFactor(4);
writer.AddIndexesNoOptimize(new Directory[] { aux, aux2 });
Assert.AreEqual(1025, writer.DocCount());
Assert.AreEqual(1000, writer.GetDocCount(0));
writer.Close();
// make sure the index is correct
VerifyNumDocs(dir, 1025);
}
private void CheckInvariants(IndexWriter writer)
{
writer.WaitForMerges();
int maxBufferedDocs = writer.GetMaxBufferedDocs();
int mergeFactor = writer.GetMergeFactor();
int maxMergeDocs = writer.GetMaxMergeDocs();
int ramSegmentCount = writer.GetNumBufferedDocuments();
Assert.IsTrue(ramSegmentCount < maxBufferedDocs);
int lowerBound = -1;
int upperBound = maxBufferedDocs;
int numSegments = 0;
int segmentCount = writer.GetSegmentCount();
for (int i = segmentCount - 1; i >= 0; i--)
{
int docCount = writer.GetDocCount(i);
Assert.IsTrue(docCount > lowerBound);
if (docCount <= upperBound)
{
numSegments++;
}
else
{
if (upperBound * mergeFactor <= maxMergeDocs)
{
Assert.IsTrue(numSegments < mergeFactor);
}
do
{
lowerBound = upperBound;
upperBound *= mergeFactor;
}while (docCount > upperBound);
numSegments = 1;
}
}
if (upperBound * mergeFactor <= maxMergeDocs)
{
Assert.IsTrue(numSegments < mergeFactor);
}
}
public virtual void TestNonRAMDelete()
{
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = (0 == pass);
Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
modifier.SetMaxBufferedDocs(2);
modifier.SetMaxBufferedDeleteTerms(2);
int id = 0;
int value_Renamed = 100;
for (int i = 0; i < 7; i++)
{
AddDoc(modifier, ++id, value_Renamed);
}
modifier.Commit();
Assert.AreEqual(0, modifier.GetNumBufferedDocuments());
Assert.IsTrue(0 < modifier.GetSegmentCount());
modifier.Commit();
IndexReader reader = IndexReader.Open(dir);
Assert.AreEqual(7, reader.NumDocs());
reader.Close();
modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
modifier.Commit();
reader = IndexReader.Open(dir);
Assert.AreEqual(0, reader.NumDocs());
reader.Close();
modifier.Close();
dir.Close();
}
}
public virtual void TestNonRAMDelete()
{
Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), true,
IndexWriter.MaxFieldLength.UNLIMITED, null);
modifier.SetMaxBufferedDocs(2);
modifier.SetMaxBufferedDeleteTerms(2);
int id = 0;
int value_Renamed = 100;
for (int i = 0; i < 7; i++)
{
AddDoc(modifier, ++id, value_Renamed);
}
modifier.Commit(null);
Assert.AreEqual(0, modifier.GetNumBufferedDocuments());
Assert.IsTrue(0 < modifier.GetSegmentCount());
modifier.Commit(null);
IndexReader reader = IndexReader.Open(dir, true, null);
Assert.AreEqual(7, reader.NumDocs());
reader.Close();
modifier.DeleteDocuments(null, new Term("value", System.Convert.ToString(value_Renamed)));
modifier.Commit(null);
reader = IndexReader.Open(dir, true, null);
Assert.AreEqual(0, reader.NumDocs());
reader.Close();
modifier.Close();
dir.Close();
}
public virtual void TestNoOverMerge()
{
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
writer.MergeFactor = 10;
writer.SetMergePolicy(new LogDocMergePolicy(writer));
bool noOverMerge = false;
for (int i = 0; i < 100; i++)
{
AddDoc(writer);
CheckInvariants(writer);
if (writer.GetNumBufferedDocuments() + writer.GetSegmentCount() >= 18)
{
noOverMerge = true;
}
}
Assert.IsTrue(noOverMerge);
writer.Close();
}
public virtual void TestNoCopySegments()
{
// main directory
Directory dir = new RAMDirectory();
// auxiliary directory
Directory aux = new RAMDirectory();
SetUpDirs(dir, aux);
IndexWriter writer = NewWriter(dir, false);
writer.SetMaxBufferedDocs(9);
writer.SetMergeFactor(4);
AddDocs(writer, 2);
writer.AddIndexesNoOptimize(new Directory[] { aux });
Assert.AreEqual(1032, writer.DocCount());
Assert.AreEqual(2, writer.GetSegmentCount());
Assert.AreEqual(1000, writer.GetDocCount(0));
writer.Close();
// make sure the index is correct
VerifyNumDocs(dir, 1032);
}
public virtual void TestRAMDeletes()
{
for (int pass = 0; pass < 2; pass++)
{
for (int t = 0; t < 2; t++)
{
bool autoCommit = (0 == pass);
Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
modifier.SetMaxBufferedDocs(4);
modifier.SetMaxBufferedDeleteTerms(4);
int id = 0;
int value_Renamed = 100;
AddDoc(modifier, ++id, value_Renamed);
if (0 == t)
modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
else
modifier.DeleteDocuments(new TermQuery(new Term("value", System.Convert.ToString(value_Renamed))));
AddDoc(modifier, ++id, value_Renamed);
if (0 == t)
{
modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
Assert.AreEqual(2, modifier.GetNumBufferedDeleteTerms());
Assert.AreEqual(1, modifier.GetBufferedDeleteTermsSize());
}
else
modifier.DeleteDocuments(new TermQuery(new Term("value", System.Convert.ToString(value_Renamed))));
AddDoc(modifier, ++id, value_Renamed);
Assert.AreEqual(0, modifier.GetSegmentCount());
modifier.Flush();
modifier.Commit();
IndexReader reader = IndexReader.Open(dir);
Assert.AreEqual(1, reader.NumDocs());
int hitCount = GetHitCount(dir, new Term("id", System.Convert.ToString(id)));
Assert.AreEqual(1, hitCount);
reader.Close();
modifier.Close();
dir.Close();
}
}
}
public virtual void TestNonRAMDelete()
{
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = (0 == pass);
Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
modifier.SetMaxBufferedDocs(2);
modifier.SetMaxBufferedDeleteTerms(2);
int id = 0;
int value_Renamed = 100;
for (int i = 0; i < 7; i++)
{
AddDoc(modifier, ++id, value_Renamed);
}
modifier.Commit();
Assert.AreEqual(0, modifier.GetNumBufferedDocuments());
Assert.IsTrue(0 < modifier.GetSegmentCount());
modifier.Commit();
IndexReader reader = IndexReader.Open(dir);
Assert.AreEqual(7, reader.NumDocs());
reader.Close();
modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
modifier.Commit();
reader = IndexReader.Open(dir);
Assert.AreEqual(0, reader.NumDocs());
reader.Close();
modifier.Close();
dir.Close();
}
}
private void CheckInvariants(IndexWriter writer)
{
writer.WaitForMerges();
int maxBufferedDocs = writer.GetMaxBufferedDocs();
int mergeFactor = writer.MergeFactor;
int maxMergeDocs = writer.MaxMergeDocs;
int ramSegmentCount = writer.GetNumBufferedDocuments();
Assert.IsTrue(ramSegmentCount < maxBufferedDocs);
int lowerBound = - 1;
int upperBound = maxBufferedDocs;
int numSegments = 0;
int segmentCount = writer.GetSegmentCount();
for (int i = segmentCount - 1; i >= 0; i--)
{
int docCount = writer.GetDocCount(i);
Assert.IsTrue(docCount > lowerBound);
if (docCount <= upperBound)
{
numSegments++;
}
else
{
if (upperBound * mergeFactor <= maxMergeDocs)
{
Assert.IsTrue(numSegments < mergeFactor, "maxMergeDocs=" + maxMergeDocs + "; numSegments=" + numSegments + "; upperBound=" + upperBound + "; mergeFactor=" + mergeFactor);
}
do
{
lowerBound = upperBound;
upperBound *= mergeFactor;
}
while (docCount > upperBound);
numSegments = 1;
}
}
if (upperBound * mergeFactor <= maxMergeDocs)
{
Assert.IsTrue(numSegments < mergeFactor);
}
}
private void CheckInvariants(IndexWriter writer)
{
_TestUtil.SyncConcurrentMerges(writer);
int maxBufferedDocs = writer.GetMaxBufferedDocs();
int mergeFactor = writer.GetMergeFactor();
int maxMergeDocs = writer.GetMaxMergeDocs();
int ramSegmentCount = writer.GetNumBufferedDocuments();
Assert.IsTrue(ramSegmentCount < maxBufferedDocs);
int lowerBound = - 1;
int upperBound = maxBufferedDocs;
int numSegments = 0;
int segmentCount = writer.GetSegmentCount();
for (int i = segmentCount - 1; i >= 0; i--)
{
int docCount = writer.GetDocCount(i);
Assert.IsTrue(docCount > lowerBound);
if (docCount <= upperBound)
{
numSegments++;
}
else
{
if (upperBound * mergeFactor <= maxMergeDocs)
{
Assert.IsTrue(numSegments < mergeFactor);
}
do
{
lowerBound = upperBound;
upperBound *= mergeFactor;
}
while (docCount > upperBound);
numSegments = 1;
}
}
if (upperBound * mergeFactor <= maxMergeDocs)
{
Assert.IsTrue(numSegments < mergeFactor);
}
System.String[] files = writer.GetDirectory().ListAll();
int segmentCfsCount = 0;
for (int i = 0; i < files.Length; i++)
{
if (files[i].EndsWith(".cfs"))
{
segmentCfsCount++;
}
}
Assert.AreEqual(segmentCount, segmentCfsCount);
}
private void PrintSegmentDocCounts(IndexWriter writer)
{
int segmentCount = writer.GetSegmentCount();
System.Console.Out.WriteLine("" + segmentCount + " segments total");
for (int i = 0; i < segmentCount; i++)
{
System.Console.Out.WriteLine(" segment " + i + " has " + writer.GetDocCount(i) + " docs");
}
}