/// <summary>
/// static boolean hasPendingDeletes(SegmentInfos infos) {
/// for (SegmentInfo info : infos) {
/// if (info.deletes.Any()) {
/// return true;
/// }
/// }
/// return false;
/// }
///
/// </summary>
internal virtual void Part2(IndexWriter writer, RangeMergePolicy fsmp)
{
for (int x = 20; x < 25; x++)
{
writer.AddDocument(DocHelper.CreateDocument(x, "5", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
writer.Flush(false, false);
for (int x = 25; x < 30; x++)
{
writer.AddDocument(DocHelper.CreateDocument(x, "5", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
writer.Flush(false, false);
//System.out.println("infos3:"+writer.SegmentInfos);
Term delterm = new Term("id", "8");
writer.DeleteDocuments(delterm);
//System.out.println("segdels3:" + writer.docWriter.deletesToString());
fsmp.DoMerge = true;
fsmp.Start = 1;
fsmp.Length = 2;
writer.MaybeMerge();
// deletes for info1, the newly created segment from the
// merge should have no deletes because they were applied in
// the merge
//SegmentInfo info1 = writer.SegmentInfos.Info(1);
//Assert.IsFalse(exists(info1, writer.docWriter.segmentDeletes));
//System.out.println("infos4:"+writer.SegmentInfos);
//System.out.println("segdels4:" + writer.docWriter.deletesToString());
}
public virtual void TestDeletes1()
{
//IndexWriter.debug2 = System.out;
Directory dir = new MockDirectoryWrapper(new J2N.Randomizer(Random.NextInt64()), new RAMDirectory());
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMergeScheduler(new SerialMergeScheduler());
iwc.SetMaxBufferedDocs(5000);
iwc.SetRAMBufferSizeMB(100);
RangeMergePolicy fsmp = new RangeMergePolicy(this, false);
iwc.SetMergePolicy(fsmp);
IndexWriter writer = new IndexWriter(dir, iwc);
for (int x = 0; x < 5; x++)
{
writer.AddDocument(DocHelper.CreateDocument(x, "1", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
//System.out.println("commit1");
writer.Commit();
Assert.AreEqual(1, writer.SegmentCount);
for (int x = 5; x < 10; x++)
{
writer.AddDocument(DocHelper.CreateDocument(x, "2", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
//System.out.println("commit2");
writer.Commit();
Assert.AreEqual(2, writer.SegmentCount);
for (int x = 10; x < 15; x++)
{
writer.AddDocument(DocHelper.CreateDocument(x, "3", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
writer.DeleteDocuments(new Term("id", "1"));
writer.DeleteDocuments(new Term("id", "11"));
// flushing without applying deletes means
// there will still be deletes in the segment infos
writer.Flush(false, false);
Assert.IsTrue(writer.bufferedUpdatesStream.Any());
// get reader flushes pending deletes
// so there should not be anymore
IndexReader r1 = writer.GetReader();
Assert.IsFalse(writer.bufferedUpdatesStream.Any());
r1.Dispose();
// delete id:2 from the first segment
// merge segments 0 and 1
// which should apply the delete id:2
writer.DeleteDocuments(new Term("id", "2"));
writer.Flush(false, false);
fsmp = (RangeMergePolicy)writer.Config.MergePolicy;
fsmp.doMerge = true;
fsmp.start = 0;
fsmp.length = 2;
writer.MaybeMerge();
Assert.AreEqual(2, writer.SegmentCount);
// id:2 shouldn't exist anymore because
// it's been applied in the merge and now it's gone
IndexReader r2 = writer.GetReader();
int[] id2docs = ToDocsArray(new Term("id", "2"), null, r2);
Assert.IsTrue(id2docs is null);
r2.Dispose();
/*
* /// // added docs are in the ram buffer
* /// for (int x = 15; x < 20; x++) {
* /// writer.AddDocument(TestIndexWriterReader.CreateDocument(x, "4", 2));
* /// System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
* /// }
* /// Assert.IsTrue(writer.numRamDocs() > 0);
* /// // delete from the ram buffer
* /// writer.DeleteDocuments(new Term("id", Integer.toString(13)));
* ///
* /// Term id3 = new Term("id", Integer.toString(3));
* ///
* /// // delete from the 1st segment
* /// writer.DeleteDocuments(id3);
* ///
* /// Assert.IsTrue(writer.numRamDocs() > 0);
* ///
* /// //System.out
* /// // .println("segdels1:" + writer.docWriter.deletesToString());
* ///
* /// //Assert.IsTrue(writer.docWriter.segmentDeletes.Size() > 0);
* ///
* /// // we cause a merge to happen
* /// fsmp.doMerge = true;
* /// fsmp.start = 0;
* /// fsmp.length = 2;
* /// System.out.println("maybeMerge "+writer.SegmentInfos);
* ///
* /// SegmentInfo info0 = writer.SegmentInfos[0];
* /// SegmentInfo info1 = writer.SegmentInfos[1];
* ///
* /// writer.MaybeMerge();
* /// System.out.println("maybeMerge after "+writer.SegmentInfos);
* /// // there should be docs in RAM
* /// Assert.IsTrue(writer.numRamDocs() > 0);
* ///
* /// // assert we've merged the 1 and 2 segments
* /// // and still have a segment leftover == 2
* /// Assert.AreEqual(2, writer.SegmentInfos.Size());
* /// Assert.IsFalse(segThere(info0, writer.SegmentInfos));
* /// Assert.IsFalse(segThere(info1, writer.SegmentInfos));
* ///
* /// //System.out.println("segdels2:" + writer.docWriter.deletesToString());
* ///
* /// //Assert.IsTrue(writer.docWriter.segmentDeletes.Size() > 0);
* ///
* /// IndexReader r = writer.GetReader();
* /// IndexReader r1 = r.getSequentialSubReaders()[0];
* /// printDelDocs(r1.GetLiveDocs());
* /// int[] docs = toDocsArray(id3, null, r);
* /// System.out.println("id3 docs:"+Arrays.toString(docs));
* /// // there shouldn't be any docs for id:3
* /// Assert.IsTrue(docs is null);
* /// r.Dispose();
* ///
* /// part2(writer, fsmp);
* ///
*/
// System.out.println("segdels2:"+writer.docWriter.segmentDeletes.toString());
//System.out.println("close");
writer.Dispose();
dir.Dispose();
}
public void Test()
{
DirectoryInfo dir = CreateTempDir(GetType().Name);
DirectoryInfo destDir = CreateTempDir(GetType().Name);
Store.Directory fsDir = NewFSDirectory(dir);
// IndexSplitter.split makes its own commit directly with SIPC/SegmentInfos,
// so the unreferenced files are expected.
if (fsDir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)fsDir).AssertNoUnrefencedFilesOnClose = (false);
}
MergePolicy mergePolicy = new LogByteSizeMergePolicy();
mergePolicy.NoCFSRatio = 1.0;
mergePolicy.MaxCFSSegmentSizeMB = double.PositiveInfinity;
IndexWriter iw = new IndexWriter(
fsDir,
new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).
SetOpenMode(OpenMode.CREATE).
SetMergePolicy(mergePolicy)
);
for (int x = 0; x < 100; x++)
{
Document doc = DocHelper.CreateDocument(x, "index", 5);
iw.AddDocument(doc);
}
iw.Commit();
for (int x = 100; x < 150; x++)
{
Document doc = DocHelper.CreateDocument(x, "index2", 5);
iw.AddDocument(doc);
}
iw.Commit();
for (int x = 150; x < 200; x++)
{
Document doc = DocHelper.CreateDocument(x, "index3", 5);
iw.AddDocument(doc);
}
iw.Commit();
DirectoryReader iwReader = iw.GetReader();
assertEquals(3, iwReader.Leaves.Count);
iwReader.Dispose();
iw.Dispose();
// we should have 2 segments now
IndexSplitter @is = new IndexSplitter(dir);
string splitSegName = @is.Infos.Info(1).Info.Name;
@is.Split(destDir, new string[] { splitSegName });
Store.Directory fsDirDest = NewFSDirectory(destDir);
DirectoryReader r = DirectoryReader.Open(fsDirDest);
assertEquals(50, r.MaxDoc);
r.Dispose();
fsDirDest.Dispose();
// now test cmdline
DirectoryInfo destDir2 = CreateTempDir(GetType().Name);
IndexSplitter.Main(new String[] { dir.FullName, destDir2.FullName, splitSegName });
assertEquals(5, destDir2.GetFiles().Length);
Store.Directory fsDirDest2 = NewFSDirectory(destDir2);
r = DirectoryReader.Open(fsDirDest2);
assertEquals(50, r.MaxDoc);
r.Dispose();
fsDirDest2.Dispose();
// now remove the copied segment from src
IndexSplitter.Main(new String[] { dir.FullName, "-d", splitSegName });
r = DirectoryReader.Open(fsDir);
assertEquals(2, r.Leaves.size());
r.Dispose();
fsDir.Dispose();
}
public override void SetUp()
{
base.SetUp();
DocHelper.SetupDoc(testDoc);
}
public virtual void TestMerge()
{
Codec codec = Codec.Default;
SegmentInfo si = new SegmentInfo(MergedDir, Constants.LUCENE_MAIN_VERSION, MergedSegment, -1, false, codec, null);
SegmentMerger merger = new SegmentMerger(Arrays.AsList <AtomicReader>(Reader1, Reader2), si, InfoStream.Default, MergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, CheckAbort.NONE, new FieldInfos.FieldNumbers(), NewIOContext(Random()), true);
MergeState mergeState = merger.Merge();
int docsMerged = mergeState.SegmentInfo.DocCount;
Assert.IsTrue(docsMerged == 2);
//Should be able to open a new SegmentReader against the new directory
SegmentReader mergedReader = new SegmentReader(new SegmentCommitInfo(new SegmentInfo(MergedDir, Constants.LUCENE_MAIN_VERSION, MergedSegment, docsMerged, false, codec, null), 0, -1L, -1L), DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, NewIOContext(Random()));
Assert.IsTrue(mergedReader != null);
Assert.IsTrue(mergedReader.NumDocs == 2);
Document newDoc1 = mergedReader.Document(0);
Assert.IsTrue(newDoc1 != null);
//There are 2 unstored fields on the document
Assert.IsTrue(DocHelper.NumFields(newDoc1) == DocHelper.NumFields(Doc1) - DocHelper.Unstored.Count);
Document newDoc2 = mergedReader.Document(1);
Assert.IsTrue(newDoc2 != null);
Assert.IsTrue(DocHelper.NumFields(newDoc2) == DocHelper.NumFields(Doc2) - DocHelper.Unstored.Count);
DocsEnum termDocs = TestUtil.Docs(Random(), mergedReader, DocHelper.TEXT_FIELD_2_KEY, new BytesRef("field"), MultiFields.GetLiveDocs(mergedReader), null, 0);
Assert.IsTrue(termDocs != null);
Assert.IsTrue(termDocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int tvCount = 0;
foreach (FieldInfo fieldInfo in mergedReader.FieldInfos)
{
if (fieldInfo.HasVectors)
{
tvCount++;
}
}
//System.out.println("stored size: " + stored.Size());
Assert.AreEqual(3, tvCount, "We do not have 3 fields that were indexed with term vector");
Terms vector = mergedReader.GetTermVectors(0).GetTerms(DocHelper.TEXT_FIELD_2_KEY);
Assert.IsNotNull(vector);
Assert.AreEqual(3, vector.Count);
TermsEnum termsEnum = vector.GetIterator(null);
int i = 0;
while (termsEnum.Next() != null)
{
string term = termsEnum.Term.Utf8ToString();
int freq = (int)termsEnum.TotalTermFreq;
//System.out.println("Term: " + term + " Freq: " + freq);
Assert.IsTrue(DocHelper.FIELD_2_TEXT.IndexOf(term) != -1);
Assert.IsTrue(DocHelper.FIELD_2_FREQS[i] == freq);
i++;
}
TestSegmentReader.CheckNorms(mergedReader);
mergedReader.Dispose();
}
public override void SetUp()
{
base.SetUp();
DocHelper.SetupDoc(testDoc);
info = DocHelper.WriteDoc(dir, testDoc);
}
protected virtual void SetUp()
{
DocHelper.SetupDoc(testDoc);
DocHelper.WriteDoc(dir, testDoc);
}
protected virtual void SetUp()
{
DocHelper.SetupDoc(testDoc);
}
