public virtual void TestForceMergeTempSpaceUsage()
{
MockDirectoryWrapper dir = NewMockDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(10).SetMergePolicy(NewLogMergePolicy()));
if (VERBOSE)
{
Console.WriteLine("TEST: config1=" + writer.Config);
}
for (int j = 0; j < 500; j++)
{
TestIndexWriter.AddDocWithIndex(writer, j);
}
int termIndexInterval = writer.Config.TermIndexInterval;
// force one extra segment w/ different doc store so
// we see the doc stores get merged
writer.Commit();
TestIndexWriter.AddDocWithIndex(writer, 500);
writer.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: start disk usage");
}
long startDiskUsage = 0;
string[] files = dir.ListAll();
for (int i = 0; i < files.Length; i++)
{
startDiskUsage += dir.FileLength(files[i]);
if (VERBOSE)
{
Console.WriteLine(files[i] + ": " + dir.FileLength(files[i]));
}
}
dir.ResetMaxUsedSizeInBytes();
dir.TrackDiskUsage = true;
// Import to use same term index interval else a
// smaller one here could increase the disk usage and
// cause a false failure:
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.APPEND).SetTermIndexInterval(termIndexInterval).SetMergePolicy(NewLogMergePolicy()));
writer.ForceMerge(1);
writer.Dispose();
long maxDiskUsage = dir.MaxUsedSizeInBytes;
Assert.IsTrue(maxDiskUsage <= 4 * startDiskUsage, "forceMerge used too much temporary space: starting usage was " + startDiskUsage + " bytes; max temp usage was " + maxDiskUsage + " but should have been " + (4 * startDiskUsage) + " (= 4X starting usage)");
dir.Dispose();
}
public virtual void TestCommitOnCloseForceMerge()
{
Directory dir = NewDirectory();
// Must disable throwing exc on double-write: this
// test uses IW.rollback which easily results in
// writing to same file more than once
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).PreventDoubleWrite = false;
}
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(10).SetMergePolicy(NewLogMergePolicy(10)));
for (int j = 0; j < 17; j++)
{
TestIndexWriter.AddDocWithIndex(writer, j);
}
writer.Dispose();
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.APPEND));
writer.ForceMerge(1);
// Open a reader before closing (commiting) the writer:
DirectoryReader reader = DirectoryReader.Open(dir);
// Reader should see index as multi-seg at this
// point:
Assert.IsTrue(reader.Leaves.Count > 1, "Reader incorrectly sees one segment");
reader.Dispose();
// Abort the writer:
writer.Rollback();
TestIndexWriter.AssertNoUnreferencedFiles(dir, "aborted writer after forceMerge");
// Open a reader after aborting writer:
reader = DirectoryReader.Open(dir);
// Reader should still see index as multi-segment
Assert.IsTrue(reader.Leaves.Count > 1, "Reader incorrectly sees one segment");
reader.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: do real full merge");
}
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.APPEND));
writer.ForceMerge(1);
writer.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: writer closed");
}
TestIndexWriter.AssertNoUnreferencedFiles(dir, "aborted writer after forceMerge");
// Open a reader after aborting writer:
reader = DirectoryReader.Open(dir);
// Reader should see index as one segment
Assert.AreEqual(1, reader.Leaves.Count, "Reader incorrectly sees more than one segment");
reader.Dispose();
dir.Dispose();
}
public virtual void TestCommitOnCloseDiskUsage()
{
// MemoryCodec, since it uses FST, is not necessarily
// "additive", ie if you add up N small FSTs, then merge
// them, the merged result can easily be larger than the
// sum because the merged FST may use array encoding for
// some arcs (which uses more space):
string idFormat = TestUtil.GetPostingsFormat("id");
string contentFormat = TestUtil.GetPostingsFormat("content");
AssumeFalse("this test cannot run with Memory codec", idFormat.Equals("Memory") || contentFormat.Equals("Memory"));
MockDirectoryWrapper dir = NewMockDirectory();
Analyzer analyzer;
if (Random().NextBoolean())
{
// no payloads
analyzer = new AnalyzerAnonymousInnerClassHelper(this);
}
else
{
// fixed length payloads
int length = Random().Next(200);
analyzer = new AnalyzerAnonymousInnerClassHelper2(this, length);
}
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(10).SetReaderPooling(false).SetMergePolicy(NewLogMergePolicy(10)));
for (int j = 0; j < 30; j++)
{
TestIndexWriter.AddDocWithIndex(writer, j);
}
writer.Dispose();
dir.ResetMaxUsedSizeInBytes();
dir.TrackDiskUsage = true;
long startDiskUsage = dir.MaxUsedSizeInBytes;
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetOpenMode(OpenMode_e.APPEND).SetMaxBufferedDocs(10).SetMergeScheduler(new SerialMergeScheduler()).SetReaderPooling(false).SetMergePolicy(NewLogMergePolicy(10)));
for (int j = 0; j < 1470; j++)
{
TestIndexWriter.AddDocWithIndex(writer, j);
}
long midDiskUsage = dir.MaxUsedSizeInBytes;
dir.ResetMaxUsedSizeInBytes();
writer.ForceMerge(1);
writer.Dispose();
DirectoryReader.Open(dir).Dispose();
long endDiskUsage = dir.MaxUsedSizeInBytes;
// Ending index is 50X as large as starting index; due
// to 3X disk usage normally we allow 150X max
// transient usage. If something is wrong w/ deleter
// and it doesn't delete intermediate segments then it
// will exceed this 150X:
// System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " + endDiskUsage);
Assert.IsTrue(midDiskUsage < 150 * startDiskUsage, "writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage + " max=" + (startDiskUsage * 150));
Assert.IsTrue(endDiskUsage < 150 * startDiskUsage, "writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage + " max=" + (startDiskUsage * 150));
dir.Dispose();
}