public virtual void TestForceMergeTempSpaceUsage()
{
MockDirectoryWrapper dir = NewMockDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(10).SetMergePolicy(NewLogMergePolicy()));
if (VERBOSE)
{
Console.WriteLine("TEST: config1=" + writer.Config);
}
for (int j = 0; j < 500; j++)
{
AddDocWithIndex(writer, j);
}
int termIndexInterval = writer.Config.TermIndexInterval;
// force one extra segment w/ different doc store so
// we see the doc stores get merged
writer.Commit();
AddDocWithIndex(writer, 500);
writer.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: start disk usage");
}
long startDiskUsage = 0;
string[] files = dir.ListAll();
for (int i = 0; i < files.Length; i++)
{
startDiskUsage += dir.FileLength(files[i]);
if (VERBOSE)
{
Console.WriteLine(files[i] + ": " + dir.FileLength(files[i]));
}
}
dir.ResetMaxUsedSizeInBytes();
dir.TrackDiskUsage = true;
// Import to use same term index interval else a
// smaller one here could increase the disk usage and
// cause a false failure:
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetTermIndexInterval(termIndexInterval).SetMergePolicy(NewLogMergePolicy()));
writer.ForceMerge(1);
writer.Dispose();
long maxDiskUsage = dir.MaxUsedSizeInBytes;
Assert.IsTrue(maxDiskUsage <= 4 * startDiskUsage, "forceMerge used too much temporary space: starting usage was " + startDiskUsage + " bytes; max temp usage was " + maxDiskUsage + " but should have been " + (4 * startDiskUsage) + " (= 4X starting usage)");
dir.Dispose();
}
public virtual void TestCommitOnCloseDiskUsage()
{
// MemoryCodec, since it uses FST, is not necessarily
// "additive", ie if you add up N small FSTs, then merge
// them, the merged result can easily be larger than the
// sum because the merged FST may use array encoding for
// some arcs (which uses more space):
string idFormat = TestUtil.GetPostingsFormat("id");
string contentFormat = TestUtil.GetPostingsFormat("content");
AssumeFalse("this test cannot run with Memory codec", idFormat.Equals("Memory", StringComparison.Ordinal) || contentFormat.Equals("Memory", StringComparison.Ordinal));
MockDirectoryWrapper dir = NewMockDirectory();
Analyzer analyzer;
if (Random().NextBoolean())
{
// no payloads
analyzer = new AnalyzerAnonymousInnerClassHelper(this);
}
else
{
// fixed length payloads
int length = Random().Next(200);
analyzer = new AnalyzerAnonymousInnerClassHelper2(this, length);
}
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(10).SetReaderPooling(false).SetMergePolicy(NewLogMergePolicy(10)));
for (int j = 0; j < 30; j++)
{
AddDocWithIndex(writer, j);
}
writer.Dispose();
dir.ResetMaxUsedSizeInBytes();
dir.TrackDiskUsage = true;
long startDiskUsage = dir.MaxUsedSizeInBytes;
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetOpenMode(OpenMode.APPEND).SetMaxBufferedDocs(10).SetMergeScheduler(new SerialMergeScheduler()).SetReaderPooling(false).SetMergePolicy(NewLogMergePolicy(10)));
for (int j = 0; j < 1470; j++)
{
AddDocWithIndex(writer, j);
}
long midDiskUsage = dir.MaxUsedSizeInBytes;
dir.ResetMaxUsedSizeInBytes();
writer.ForceMerge(1);
writer.Dispose();
DirectoryReader.Open(dir).Dispose();
long endDiskUsage = dir.MaxUsedSizeInBytes;
// Ending index is 50X as large as starting index; due
// to 3X disk usage normally we allow 150X max
// transient usage. If something is wrong w/ deleter
// and it doesn't delete intermediate segments then it
// will exceed this 150X:
// System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " + endDiskUsage);
Assert.IsTrue(midDiskUsage < 150 * startDiskUsage, "writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage + " max=" + (startDiskUsage * 150));
Assert.IsTrue(endDiskUsage < 150 * startDiskUsage, "writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage + " max=" + (startDiskUsage * 150));
dir.Dispose();
}
