public virtual void TestForceMergeDeletes3()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetMergePolicy(NewLogMergePolicy(50)));
FieldType customType = new FieldType();
customType.IsStored = true;
FieldType customType1 = new FieldType(TextField.TYPE_NOT_STORED);
customType1.IsTokenized = false;
customType1.StoreTermVectors = true;
customType1.StoreTermVectorPositions = true;
customType1.StoreTermVectorOffsets = true;
Document document = new Document();
Field storedField = NewField("stored", "stored", customType);
document.Add(storedField);
Field termVectorField = NewField("termVector", "termVector", customType1);
document.Add(termVectorField);
Field idField = NewStringField("id", "", Field.Store.NO);
document.Add(idField);
for (int i = 0; i < 98; i++)
{
idField.SetStringValue("" + i);
writer.AddDocument(document);
}
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
Assert.AreEqual(98, ir.MaxDoc);
Assert.AreEqual(98, ir.NumDocs);
ir.Dispose();
IndexWriterConfig dontMergeConfig = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))).SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
writer = new IndexWriter(dir, dontMergeConfig);
for (int i = 0; i < 98; i += 2)
{
writer.DeleteDocuments(new Term("id", "" + i));
}
writer.Dispose();
ir = DirectoryReader.Open(dir);
Assert.AreEqual(49, ir.NumDocs);
ir.Dispose();
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NewLogMergePolicy(3)));
writer.ForceMergeDeletes(false);
writer.Dispose();
ir = DirectoryReader.Open(dir);
Assert.AreEqual(49, ir.MaxDoc);
Assert.AreEqual(49, ir.NumDocs);
ir.Dispose();
dir.Dispose();
}
public virtual void TestForceMergeDeletes()
{
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
TieredMergePolicy tmp = NewTieredMergePolicy();
conf.SetMergePolicy(tmp);
conf.SetMaxBufferedDocs(4);
tmp.MaxMergeAtOnce = 100;
tmp.SegmentsPerTier = 100;
tmp.ForceMergeDeletesPctAllowed = 30.0;
IndexWriter w = new IndexWriter(dir, conf);
for (int i = 0; i < 80; i++)
{
Document doc = new Document();
doc.Add(NewTextField("content", "aaa " + (i % 4), Field.Store.NO));
w.AddDocument(doc);
}
Assert.AreEqual(80, w.MaxDoc);
Assert.AreEqual(80, w.NumDocs);
if (VERBOSE)
{
Console.WriteLine("\nTEST: delete docs");
}
w.DeleteDocuments(new Term("content", "0"));
w.ForceMergeDeletes();
Assert.AreEqual(80, w.MaxDoc);
Assert.AreEqual(60, w.NumDocs);
if (VERBOSE)
{
Console.WriteLine("\nTEST: forceMergeDeletes2");
}
((TieredMergePolicy)w.Config.MergePolicy).ForceMergeDeletesPctAllowed = 10.0;
w.ForceMergeDeletes();
Assert.AreEqual(60, w.NumDocs);
Assert.AreEqual(60, w.MaxDoc);
w.Dispose();
dir.Dispose();
}
public virtual void TestForceMergeDeletes()
{
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
TieredMergePolicy tmp = NewTieredMergePolicy();
conf.SetMergePolicy(tmp);
conf.SetMaxBufferedDocs(4);
tmp.MaxMergeAtOnce = 100;
tmp.SegmentsPerTier = 100;
tmp.ForceMergeDeletesPctAllowed = 30.0;
IndexWriter w = new IndexWriter(dir, conf);
for (int i = 0; i < 80; i++)
{
Document doc = new Document();
doc.Add(NewTextField("content", "aaa " + (i % 4), Field.Store.NO));
w.AddDocument(doc);
}
Assert.AreEqual(80, w.MaxDoc);
Assert.AreEqual(80, w.NumDocs());
if (VERBOSE)
{
Console.WriteLine("\nTEST: delete docs");
}
w.DeleteDocuments(new Term("content", "0"));
w.ForceMergeDeletes();
Assert.AreEqual(80, w.MaxDoc);
Assert.AreEqual(60, w.NumDocs());
if (VERBOSE)
{
Console.WriteLine("\nTEST: forceMergeDeletes2");
}
((TieredMergePolicy)w.Config.MergePolicy).ForceMergeDeletesPctAllowed = 10.0;
w.ForceMergeDeletes();
Assert.AreEqual(60, w.NumDocs());
Assert.AreEqual(60, w.MaxDoc);
w.Dispose();
dir.Dispose();
}
public virtual void TestForceMergeDeletes3()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetMergePolicy(NewLogMergePolicy(50)));
FieldType customType = new FieldType();
customType.Stored = true;
FieldType customType1 = new FieldType(TextField.TYPE_NOT_STORED);
customType1.Tokenized = false;
customType1.StoreTermVectors = true;
customType1.StoreTermVectorPositions = true;
customType1.StoreTermVectorOffsets = true;
Document document = new Document();
Field storedField = NewField("stored", "stored", customType);
document.Add(storedField);
Field termVectorField = NewField("termVector", "termVector", customType1);
document.Add(termVectorField);
Field idField = NewStringField("id", "", Field.Store.NO);
document.Add(idField);
for (int i = 0; i < 98; i++)
{
idField.StringValue = "" + i;
writer.AddDocument(document);
}
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
Assert.AreEqual(98, ir.MaxDoc());
Assert.AreEqual(98, ir.NumDocs());
ir.Dispose();
IndexWriterConfig dontMergeConfig = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
writer = new IndexWriter(dir, dontMergeConfig);
for (int i = 0; i < 98; i += 2)
{
writer.DeleteDocuments(new Term("id", "" + i));
}
writer.Dispose();
ir = DirectoryReader.Open(dir);
Assert.AreEqual(49, ir.NumDocs());
ir.Dispose();
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy(3)));
writer.ForceMergeDeletes(false);
writer.Dispose();
ir = DirectoryReader.Open(dir);
Assert.AreEqual(49, ir.MaxDoc());
Assert.AreEqual(49, ir.NumDocs());
ir.Dispose();
dir.Dispose();
}
public virtual void ForceMergeDeletes(bool doWait) => IndexWriter.ForceMergeDeletes(doWait);
public virtual void TestFieldNumberGaps()
{
int numIters = AtLeast(13);
for (int i = 0; i < numIters; i++)
{
Directory dir = NewDirectory();
{
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES));
Document d = new Document();
d.Add(new TextField("f1", "d1 first field", Field.Store.YES));
d.Add(new TextField("f2", "d1 second field", Field.Store.YES));
writer.AddDocument(d);
writer.Dispose();
SegmentInfos sis = new SegmentInfos();
sis.Read(dir);
Assert.AreEqual(1, sis.Count);
FieldInfos fis1 = SegmentReader.ReadFieldInfos(sis.Info(0));
Assert.AreEqual("f1", fis1.FieldInfo(0).Name);
Assert.AreEqual("f2", fis1.FieldInfo(1).Name);
}
{
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(Random.NextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES : NoMergePolicy.COMPOUND_FILES));
Document d = new Document();
d.Add(new TextField("f1", "d2 first field", Field.Store.YES));
d.Add(new StoredField("f3", new byte[] { 1, 2, 3 }));
writer.AddDocument(d);
writer.Dispose();
SegmentInfos sis = new SegmentInfos();
sis.Read(dir);
Assert.AreEqual(2, sis.Count);
FieldInfos fis1 = SegmentReader.ReadFieldInfos(sis.Info(0));
FieldInfos fis2 = SegmentReader.ReadFieldInfos(sis.Info(1));
Assert.AreEqual("f1", fis1.FieldInfo(0).Name);
Assert.AreEqual("f2", fis1.FieldInfo(1).Name);
Assert.AreEqual("f1", fis2.FieldInfo(0).Name);
Assert.IsNull(fis2.FieldInfo(1));
Assert.AreEqual("f3", fis2.FieldInfo(2).Name);
}
{
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(Random.NextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES : NoMergePolicy.COMPOUND_FILES));
Document d = new Document();
d.Add(new TextField("f1", "d3 first field", Field.Store.YES));
d.Add(new TextField("f2", "d3 second field", Field.Store.YES));
d.Add(new StoredField("f3", new byte[] { 1, 2, 3, 4, 5 }));
writer.AddDocument(d);
writer.Dispose();
SegmentInfos sis = new SegmentInfos();
sis.Read(dir);
Assert.AreEqual(3, sis.Count);
FieldInfos fis1 = SegmentReader.ReadFieldInfos(sis.Info(0));
FieldInfos fis2 = SegmentReader.ReadFieldInfos(sis.Info(1));
FieldInfos fis3 = SegmentReader.ReadFieldInfos(sis.Info(2));
Assert.AreEqual("f1", fis1.FieldInfo(0).Name);
Assert.AreEqual("f2", fis1.FieldInfo(1).Name);
Assert.AreEqual("f1", fis2.FieldInfo(0).Name);
Assert.IsNull(fis2.FieldInfo(1));
Assert.AreEqual("f3", fis2.FieldInfo(2).Name);
Assert.AreEqual("f1", fis3.FieldInfo(0).Name);
Assert.AreEqual("f2", fis3.FieldInfo(1).Name);
Assert.AreEqual("f3", fis3.FieldInfo(2).Name);
}
{
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(Random.NextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES : NoMergePolicy.COMPOUND_FILES));
writer.DeleteDocuments(new Term("f1", "d1"));
// nuke the first segment entirely so that the segment with gaps is
// loaded first!
writer.ForceMergeDeletes();
writer.Dispose();
}
IndexWriter writer_ = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(new LogByteSizeMergePolicy()).SetInfoStream(new FailOnNonBulkMergesInfoStream()));
writer_.ForceMerge(1);
writer_.Dispose();
SegmentInfos sis_ = new SegmentInfos();
sis_.Read(dir);
Assert.AreEqual(1, sis_.Count);
FieldInfos fis1_ = SegmentReader.ReadFieldInfos(sis_.Info(0));
Assert.AreEqual("f1", fis1_.FieldInfo(0).Name);
Assert.AreEqual("f2", fis1_.FieldInfo(1).Name);
Assert.AreEqual("f3", fis1_.FieldInfo(2).Name);
dir.Dispose();
}
}
public virtual void TestForceMergeDeletes()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));
Document doc = new Document();
doc.Add(NewTextField("field", "a b c", Field.Store.NO));
Field id = NewStringField("id", "", Field.Store.NO);
doc.Add(id);
id.StringValue = "0";
w.AddDocument(doc);
id.StringValue = "1";
w.AddDocument(doc);
w.DeleteDocuments(new Term("id", "0"));
IndexReader r = w.Reader;
w.ForceMergeDeletes();
w.Dispose();
r.Dispose();
r = DirectoryReader.Open(dir);
Assert.AreEqual(1, r.NumDocs());
Assert.IsFalse(r.HasDeletions());
r.Dispose();
dir.Dispose();
}
public virtual void ForceMergeDeletes()
{
IndexWriter.ForceMergeDeletes();
}
public virtual void TestFieldNumberGaps()
{
int numIters = AtLeast(13);
for (int i = 0; i < numIters; i++)
{
Directory dir = NewDirectory();
{
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES));
Document d = new Document();
d.Add(new TextField("f1", "d1 first field", Field.Store.YES));
d.Add(new TextField("f2", "d1 second field", Field.Store.YES));
writer.AddDocument(d);
writer.Dispose();
SegmentInfos sis = new SegmentInfos();
sis.Read(dir);
Assert.AreEqual(1, sis.Size());
FieldInfos fis1 = SegmentReader.ReadFieldInfos(sis.Info(0));
Assert.AreEqual("f1", fis1.FieldInfo(0).Name);
Assert.AreEqual("f2", fis1.FieldInfo(1).Name);
}
{
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES : NoMergePolicy.COMPOUND_FILES));
Document d = new Document();
d.Add(new TextField("f1", "d2 first field", Field.Store.YES));
d.Add(new StoredField("f3", new byte[] { 1, 2, 3 }));
writer.AddDocument(d);
writer.Dispose();
SegmentInfos sis = new SegmentInfos();
sis.Read(dir);
Assert.AreEqual(2, sis.Size());
FieldInfos fis1 = SegmentReader.ReadFieldInfos(sis.Info(0));
FieldInfos fis2 = SegmentReader.ReadFieldInfos(sis.Info(1));
Assert.AreEqual("f1", fis1.FieldInfo(0).Name);
Assert.AreEqual("f2", fis1.FieldInfo(1).Name);
Assert.AreEqual("f1", fis2.FieldInfo(0).Name);
Assert.IsNull(fis2.FieldInfo(1));
Assert.AreEqual("f3", fis2.FieldInfo(2).Name);
}
{
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES : NoMergePolicy.COMPOUND_FILES));
Document d = new Document();
d.Add(new TextField("f1", "d3 first field", Field.Store.YES));
d.Add(new TextField("f2", "d3 second field", Field.Store.YES));
d.Add(new StoredField("f3", new byte[] { 1, 2, 3, 4, 5 }));
writer.AddDocument(d);
writer.Dispose();
SegmentInfos sis = new SegmentInfos();
sis.Read(dir);
Assert.AreEqual(3, sis.Size());
FieldInfos fis1 = SegmentReader.ReadFieldInfos(sis.Info(0));
FieldInfos fis2 = SegmentReader.ReadFieldInfos(sis.Info(1));
FieldInfos fis3 = SegmentReader.ReadFieldInfos(sis.Info(2));
Assert.AreEqual("f1", fis1.FieldInfo(0).Name);
Assert.AreEqual("f2", fis1.FieldInfo(1).Name);
Assert.AreEqual("f1", fis2.FieldInfo(0).Name);
Assert.IsNull(fis2.FieldInfo(1));
Assert.AreEqual("f3", fis2.FieldInfo(2).Name);
Assert.AreEqual("f1", fis3.FieldInfo(0).Name);
Assert.AreEqual("f2", fis3.FieldInfo(1).Name);
Assert.AreEqual("f3", fis3.FieldInfo(2).Name);
}
{
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES : NoMergePolicy.COMPOUND_FILES));
writer.DeleteDocuments(new Term("f1", "d1"));
// nuke the first segment entirely so that the segment with gaps is
// loaded first!
writer.ForceMergeDeletes();
writer.Dispose();
}
IndexWriter writer_ = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(new LogByteSizeMergePolicy()).SetInfoStream(new FailOnNonBulkMergesInfoStream()));
writer_.ForceMerge(1);
writer_.Dispose();
SegmentInfos sis_ = new SegmentInfos();
sis_.Read(dir);
Assert.AreEqual(1, sis_.Size());
FieldInfos fis1_ = SegmentReader.ReadFieldInfos(sis_.Info(0));
Assert.AreEqual("f1", fis1_.FieldInfo(0).Name);
Assert.AreEqual("f2", fis1_.FieldInfo(1).Name);
Assert.AreEqual("f3", fis1_.FieldInfo(2).Name);
dir.Dispose();
}
}
