public virtual void TestTypeChangeViaAddIndexesIR2()
{
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)conf.Clone());
Document doc = new Document();
doc.Add(new NumericDocValuesField("dv", 0L));
writer.AddDocument(doc);
writer.Dispose();
Directory dir2 = NewDirectory();
writer = new IndexWriter(dir2, (IndexWriterConfig)conf.Clone());
IndexReader[] readers = new IndexReader[] { DirectoryReader.Open(dir) };
writer.AddIndexes(readers);
readers[0].Dispose();
doc = new Document();
doc.Add(new SortedDocValuesField("dv", new BytesRef("foo")));
try
{
writer.AddDocument(doc);
Assert.Fail("did not hit exception");
}
#pragma warning disable 168
catch (ArgumentException iae)
#pragma warning restore 168
{
// expected
}
writer.Dispose();
dir2.Dispose();
dir.Dispose();
}
private void CreateIndex(IndexWriterConfig config, Directory target, IndexReader reader, Filter preserveFilter, bool negateFilter)
{
bool success = false;
IndexWriter w = new IndexWriter(target, config);
try
{
IList <AtomicReaderContext> leaves = reader.Leaves;
IndexReader[] subReaders = new IndexReader[leaves.Count];
int i = 0;
foreach (AtomicReaderContext ctx in leaves)
{
subReaders[i++] = new DocumentFilteredAtomicIndexReader(ctx, preserveFilter, negateFilter);
}
w.AddIndexes(subReaders);
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(w);
}
else
{
IOUtils.DisposeWhileHandlingException(w);
}
}
}
public static void Main(string[] args)
{
if (args.Length < 3)
{
Console.Error.WriteLine("Usage: IndexMergeTool <mergedIndex> <index1> <index2> [index3] ...");
Environment.Exit(1);
}
FSDirectory mergedIndex = FSDirectory.Open(new System.IO.DirectoryInfo(args[0]));
#pragma warning disable 612, 618
using (IndexWriter writer = new IndexWriter(mergedIndex, new IndexWriterConfig(LuceneVersion.LUCENE_CURRENT, null)
.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE)))
#pragma warning restore 612, 618
{
Directory[] indexes = new Directory[args.Length - 1];
for (int i = 1; i < args.Length; i++)
{
indexes[i - 1] = FSDirectory.Open(new System.IO.DirectoryInfo(args[i]));
}
Console.WriteLine("Merging...");
writer.AddIndexes(indexes);
Console.WriteLine("Full merge...");
writer.ForceMerge(1);
}
Console.WriteLine("Done.");
}
public virtual void TestLucene()
{
int num = 100;
Directory indexA = NewDirectory();
Directory indexB = NewDirectory();
FillIndex(Random(), indexA, 0, num);
bool fail = VerifyIndex(indexA, 0);
if (fail)
{
Assert.Fail("Index a is invalid");
}
FillIndex(Random(), indexB, num, num);
fail = VerifyIndex(indexB, num);
if (fail)
{
Assert.Fail("Index b is invalid");
}
Directory merged = NewDirectory();
IndexWriter writer = new IndexWriter(merged, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy(2)));
writer.AddIndexes(indexA, indexB);
writer.ForceMerge(1);
writer.Dispose();
fail = VerifyIndex(merged, 0);
Assert.IsFalse(fail, "The merged index is invalid");
indexA.Dispose();
indexB.Dispose();
merged.Dispose();
}
public virtual void TestLucene()
{
int num = 100;
Directory indexA = new MockRAMDirectory();
Directory indexB = new MockRAMDirectory();
FillIndex(indexA, 0, num);
bool fail = VerifyIndex(indexA, 0);
if (fail)
{
Assert.Fail("Index a is invalid");
}
FillIndex(indexB, num, num);
fail = VerifyIndex(indexB, num);
if (fail)
{
Assert.Fail("Index b is invalid");
}
Directory merged = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(merged, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMergeFactor(2);
writer.AddIndexes(new Directory[]{indexA, indexB});
writer.Close();
fail = VerifyIndex(merged, 0);
merged.Close();
Assert.IsFalse(fail, "The merged index is invalid");
}
public virtual void TestTypeChangeViaAddIndexes2()
{
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)conf.Clone());
Document doc = new Document();
doc.Add(new NumericDocValuesField("dv", 0L));
writer.AddDocument(doc);
writer.Dispose();
Directory dir2 = NewDirectory();
writer = new IndexWriter(dir2, (IndexWriterConfig)conf.Clone());
writer.AddIndexes(dir);
doc = new Document();
doc.Add(new SortedDocValuesField("dv", new BytesRef("foo")));
try
{
writer.AddDocument(doc);
Assert.Fail("did not hit exception");
}
catch (System.ArgumentException iae)
{
// expected
}
writer.Dispose();
dir2.Dispose();
dir.Dispose();
}
public void hb()
{
Directory d = FSDirectory.GetDirectory(this.textBox1.Text, false);
Directory directory = FSDirectory.GetDirectory(this.textBox2.Text, false);
IndexWriter writer = new IndexWriter(d, new KTDictSegAnalyzer(), false);
writer.AddIndexes(new Directory[] { directory });
writer.Close();
}
public virtual void TestMixedTypesViaAddIndexes()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document doc = new Document();
doc.Add(new NumericDocValuesField("foo", 0));
w.AddDocument(doc);
// Make 2nd index w/ inconsistent field
Directory dir2 = NewDirectory();
IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
doc = new Document();
doc.Add(new SortedDocValuesField("foo", new BytesRef("hello")));
w2.AddDocument(doc);
w2.Dispose();
try
{
w.AddIndexes(dir2);
}
#pragma warning disable 168
catch (System.ArgumentException iae)
#pragma warning restore 168
{
// expected
}
IndexReader r = DirectoryReader.Open(dir2);
try
{
w.AddIndexes(new IndexReader[] { r });
}
#pragma warning disable 168
catch (System.ArgumentException iae)
#pragma warning restore 168
{
// expected
}
r.Dispose();
dir2.Dispose();
w.Dispose();
dir.Dispose();
}
public virtual void TestFilterIndexReader()
{
Directory directory = NewDirectory();
IndexWriter writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
Document d1 = new Document();
d1.Add(NewTextField("default", "one two", Field.Store.YES));
writer.AddDocument(d1);
Document d2 = new Document();
d2.Add(NewTextField("default", "one three", Field.Store.YES));
writer.AddDocument(d2);
Document d3 = new Document();
d3.Add(NewTextField("default", "two four", Field.Store.YES));
writer.AddDocument(d3);
writer.Dispose();
Directory target = NewDirectory();
// We mess with the postings so this can fail:
((BaseDirectoryWrapper)target).CrossCheckTermVectorsOnDispose = false;
writer = new IndexWriter(target, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
IndexReader reader = new TestReader(DirectoryReader.Open(directory));
writer.AddIndexes(reader);
writer.Dispose();
reader.Dispose();
reader = DirectoryReader.Open(target);
TermsEnum terms = MultiFields.GetTerms(reader, "default").GetEnumerator();
while (terms.MoveNext())
{
Assert.IsTrue(terms.Term.Utf8ToString().IndexOf('e') != -1);
}
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, terms.SeekCeil(new BytesRef("one")));
DocsAndPositionsEnum positions = terms.DocsAndPositions(MultiFields.GetLiveDocs(reader), null);
while (positions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
Assert.IsTrue((positions.DocID % 2) == 1);
}
reader.Dispose();
directory.Dispose();
target.Dispose();
}
public virtual void TestTermVectorCorruption()
{
// LUCENENET specific - log the current locking strategy used and HResult values
// for assistance troubleshooting problems on Linux/macOS
LogNativeFSFactoryDebugInfo();
Directory dir = NewDirectory();
for (int iter = 0; iter < 2; iter++)
{
IndexWriter writer = new IndexWriter(dir, ((IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)).SetMergeScheduler(new SerialMergeScheduler()).SetMergePolicy(new LogDocMergePolicy()));
Document document = new Document();
FieldType customType = new FieldType();
customType.IsStored = true;
Field storedField = NewField("stored", "stored", customType);
document.Add(storedField);
writer.AddDocument(document);
writer.AddDocument(document);
document = new Document();
document.Add(storedField);
FieldType customType2 = new FieldType(StringField.TYPE_NOT_STORED);
customType2.StoreTermVectors = true;
customType2.StoreTermVectorPositions = true;
customType2.StoreTermVectorOffsets = true;
Field termVectorField = NewField("termVector", "termVector", customType2);
document.Add(termVectorField);
writer.AddDocument(document);
writer.ForceMerge(1);
writer.Dispose();
IndexReader reader = DirectoryReader.Open(dir);
for (int i = 0; i < reader.NumDocs; i++)
{
reader.Document(i);
reader.GetTermVectors(i);
}
reader.Dispose();
writer = new IndexWriter(dir, ((IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)).SetMergeScheduler(new SerialMergeScheduler()).SetMergePolicy(new LogDocMergePolicy()));
Directory[] indexDirs = new Directory[] { new MockDirectoryWrapper(Random, new RAMDirectory(dir, NewIOContext(Random))) };
writer.AddIndexes(indexDirs);
writer.ForceMerge(1);
writer.Dispose();
}
dir.Dispose();
}
public virtual void TestEmptyIndex()
{
Directory rd1 = NewDirectory();
IndexWriter iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
iw.Dispose();
// create a copy:
Directory rd2 = NewDirectory(rd1);
Directory rdOut = NewDirectory();
IndexWriter iwOut = new IndexWriter(rdOut, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
ParallelAtomicReader apr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(rd1)), SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(rd2)));
// When unpatched, Lucene crashes here with a NoSuchElementException (caused by ParallelTermEnum)
iwOut.AddIndexes(apr);
iwOut.ForceMerge(1);
// 2nd try with a readerless parallel reader
iwOut.AddIndexes(new ParallelAtomicReader());
iwOut.ForceMerge(1);
ParallelCompositeReader cpr = new ParallelCompositeReader(DirectoryReader.Open(rd1), DirectoryReader.Open(rd2));
// When unpatched, Lucene crashes here with a NoSuchElementException (caused by ParallelTermEnum)
iwOut.AddIndexes(cpr);
iwOut.ForceMerge(1);
// 2nd try with a readerless parallel reader
iwOut.AddIndexes(new ParallelCompositeReader());
iwOut.ForceMerge(1);
iwOut.Dispose();
rdOut.Dispose();
rd1.Dispose();
rd2.Dispose();
}
public virtual void TestFilterIndexReader()
{
Directory directory = NewDirectory();
IndexWriter writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document d1 = new Document();
d1.Add(NewTextField("default", "one two", Field.Store.YES));
writer.AddDocument(d1);
Document d2 = new Document();
d2.Add(NewTextField("default", "one three", Field.Store.YES));
writer.AddDocument(d2);
Document d3 = new Document();
d3.Add(NewTextField("default", "two four", Field.Store.YES));
writer.AddDocument(d3);
writer.Dispose();
Directory target = NewDirectory();
// We mess with the postings so this can fail:
((BaseDirectoryWrapper)target).CrossCheckTermVectorsOnClose = false;
writer = new IndexWriter(target, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
IndexReader reader = new TestReader(DirectoryReader.Open(directory));
writer.AddIndexes(reader);
writer.Dispose();
reader.Dispose();
reader = DirectoryReader.Open(target);
TermsEnum terms = MultiFields.GetTerms(reader, "default").Iterator(null);
while (terms.Next() != null)
{
Assert.IsTrue(terms.Term().Utf8ToString().IndexOf('e') != -1);
}
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, terms.SeekCeil(new BytesRef("one")));
DocsAndPositionsEnum positions = terms.DocsAndPositions(MultiFields.GetLiveDocs(reader), null);
while (positions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
Assert.IsTrue((positions.DocID() % 2) == 1);
}
reader.Dispose();
directory.Dispose();
target.Dispose();
}
internal virtual void DoBody(int j, Directory[] dirs)
{
switch (j % 4)
{
case 0:
mainWriter.AddIndexes(dirs);
break;
case 1:
mainWriter.AddIndexesNoOptimize(dirs);
numAddIndexesNoOptimize.IncrementAndGet();
break;
case 2:
mainWriter.AddIndexes(readers);
break;
case 3:
mainWriter.Commit();
break;
}
count.AddAndGet(dirs.Length * NUM_INIT_DOCS);
}
public virtual void TestAddIndexes()
{
Directory dir1 = NewDirectory();
Directory dir2 = NewDirectory();
IndexWriter writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NoMergePolicy.COMPOUND_FILES));
Document d1 = new Document();
d1.Add(new TextField("f1", "first field", Field.Store.YES));
d1.Add(new TextField("f2", "second field", Field.Store.YES));
writer.AddDocument(d1);
writer.Dispose();
writer = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NoMergePolicy.COMPOUND_FILES));
Document d2 = new Document();
FieldType customType2 = new FieldType(TextField.TYPE_STORED);
customType2.StoreTermVectors = true;
d2.Add(new TextField("f2", "second field", Field.Store.YES));
d2.Add(new Field("f1", "first field", customType2));
d2.Add(new TextField("f3", "third field", Field.Store.YES));
d2.Add(new TextField("f4", "fourth field", Field.Store.YES));
writer.AddDocument(d2);
writer.Dispose();
writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NoMergePolicy.COMPOUND_FILES));
writer.AddIndexes(dir2);
writer.Dispose();
SegmentInfos sis = new SegmentInfos();
sis.Read(dir1);
Assert.AreEqual(2, sis.Count);
FieldInfos fis1 = SegmentReader.ReadFieldInfos(sis.Info(0));
FieldInfos fis2 = SegmentReader.ReadFieldInfos(sis.Info(1));
Assert.AreEqual("f1", fis1.FieldInfo(0).Name);
Assert.AreEqual("f2", fis1.FieldInfo(1).Name);
// make sure the ordering of the "external" segment is preserved
Assert.AreEqual("f2", fis2.FieldInfo(0).Name);
Assert.AreEqual("f1", fis2.FieldInfo(1).Name);
Assert.AreEqual("f3", fis2.FieldInfo(2).Name);
Assert.AreEqual("f4", fis2.FieldInfo(3).Name);
dir1.Dispose();
dir2.Dispose();
}
public virtual void TestEmptyIndex()
{
Directory rd1 = NewDirectory();
IndexWriter iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
iw.Dispose();
// create a copy:
Directory rd2 = NewDirectory(rd1);
Directory rdOut = NewDirectory();
IndexWriter iwOut = new IndexWriter(rdOut, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
ParallelAtomicReader apr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(rd1)), SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(rd2)));
// When unpatched, Lucene crashes here with a NoSuchElementException (caused by ParallelTermEnum)
iwOut.AddIndexes(apr);
iwOut.ForceMerge(1);
// 2nd try with a readerless parallel reader
iwOut.AddIndexes(new ParallelAtomicReader());
iwOut.ForceMerge(1);
ParallelCompositeReader cpr = new ParallelCompositeReader(DirectoryReader.Open(rd1), DirectoryReader.Open(rd2));
// When unpatched, Lucene crashes here with a NoSuchElementException (caused by ParallelTermEnum)
iwOut.AddIndexes(cpr);
iwOut.ForceMerge(1);
// 2nd try with a readerless parallel reader
iwOut.AddIndexes(new ParallelCompositeReader());
iwOut.ForceMerge(1);
iwOut.Dispose();
rdOut.Dispose();
rd1.Dispose();
rd2.Dispose();
}
public virtual void TestAddOldIndexesReader()
{
foreach (string name in OldNames)
{
IndexReader reader = DirectoryReader.Open(OldIndexDirs[name]);
Directory targetDir = NewDirectory();
IndexWriter w = new IndexWriter(targetDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
w.AddIndexes(reader);
w.Dispose();
reader.Dispose();
targetDir.Dispose();
}
}
private void button1_Click(object sender, EventArgs e)
{
//FilterData.PrepareCharMap();
int total = this.databaseDataSet.trans.Count;
int counter = 1;
string fsPath = indexpath;
if (!System.IO.Directory.Exists(fsPath)) System.IO.Directory.CreateDirectory(fsPath);
if (IndexReader.IndexExists(fsPath)) return;
RAMDirectory dir = new RAMDirectory();
IndexWriter ramWriter = new IndexWriter(dir, new DiacriticAnalyzer(FilterData.stopWords), true);
IndexWriter fsWriter = new IndexWriter(fsPath, new DiacriticAnalyzer(FilterData.stopWords), !IndexReader.IndexExists(fsPath));
ramWriter.SetUseCompoundFile(false);
fsWriter.SetUseCompoundFile(false);
foreach (DataRow row in this.databaseDataSet.trans.Rows)
{
Document doc = new Document();
string pid = row[this.databaseDataSet.trans.pidColumn].ToString();
string sid = row[this.databaseDataSet.trans.sidColumn].ToString();
string ayatno = row[this.databaseDataSet.trans.ayatnoColumn].ToString();
string arabic = row[this.databaseDataSet.trans.ayat_arabicColumn].ToString();
string urdu = row[this.databaseDataSet.trans.ayat_urduColumn].ToString();
string english = row[this.databaseDataSet.trans.ayat_descColumn].ToString();
doc.Add(Field.Keyword("pid", pid));
doc.Add(Field.Keyword("sid", sid));
doc.Add(Field.Keyword("ayatno", ayatno));
doc.Add(Field.Text("ayat_desc", english));
doc.Add(Field.Text("ayat_arabic", arabic));
doc.Add(Field.Text("ayat_urdu", urdu));
doc.Add(Field.Text("contents", arabic + Environment.NewLine + urdu + Environment.NewLine + english));
ramWriter.AddDocument(doc);
int percent = counter * 100 / total;
this.progressBar1.Value = percent;
label1.Text = percent.ToString() + "%";
counter++;
Application.DoEvents();
}
ramWriter.Optimize();
fsWriter.AddIndexes(new Lucene.Net.Store.Directory[] { dir });
ramWriter.Close();
fsWriter.Close();
MessageBox.Show("Done Indexing!");
}
public virtual void TestUpgradeOldSingleSegmentIndexWithAdditions()
{
foreach (string name in OldSingleSegmentNames)
{
if (VERBOSE)
{
Console.WriteLine("testUpgradeOldSingleSegmentIndexWithAdditions: index=" + name);
}
Directory dir = NewDirectory(OldIndexDirs[name]);
Assert.AreEqual(1, GetNumberOfSegments(dir), "Original index must be single segment");
// create a bunch of dummy segments
int id = 40;
RAMDirectory ramDir = new RAMDirectory();
for (int i = 0; i < 3; i++)
{
// only use Log- or TieredMergePolicy, to make document addition predictable and not suddenly merge:
MergePolicy mp = Random().NextBoolean() ? (MergePolicy)NewLogMergePolicy() : NewTieredMergePolicy();
IndexWriterConfig iwc = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMergePolicy(mp);
IndexWriter w = new IndexWriter(ramDir, iwc);
// add few more docs:
for (int j = 0; j < RANDOM_MULTIPLIER * Random().Next(30); j++)
{
AddDoc(w, id++);
}
w.Dispose(false);
}
// add dummy segments (which are all in current
// version) to single segment index
MergePolicy mp_ = Random().NextBoolean() ? (MergePolicy)NewLogMergePolicy() : NewTieredMergePolicy();
IndexWriterConfig iwc_ = (new IndexWriterConfig(TEST_VERSION_CURRENT, null)).SetMergePolicy(mp_);
IndexWriter iw = new IndexWriter(dir, iwc_);
iw.AddIndexes(ramDir);
iw.Dispose(false);
// determine count of segments in modified index
int origSegCount = GetNumberOfSegments(dir);
NewIndexUpgrader(dir).Upgrade();
int segCount = CheckAllSegmentsUpgraded(dir);
Assert.AreEqual(origSegCount, segCount, "Index must still contain the same number of segments, as only one segment was upgraded and nothing else merged");
dir.Dispose();
}
}
public virtual void TestTermVectorCorruption()
{
Directory dir = NewDirectory();
for (int iter = 0; iter < 2; iter++)
{
IndexWriter writer = new IndexWriter(dir, ((IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)).SetMergeScheduler(new SerialMergeScheduler()).SetMergePolicy(new LogDocMergePolicy()));
Document document = new Document();
FieldType customType = new FieldType();
customType.Stored = true;
Field storedField = NewField("stored", "stored", customType);
document.Add(storedField);
writer.AddDocument(document);
writer.AddDocument(document);
document = new Document();
document.Add(storedField);
FieldType customType2 = new FieldType(StringField.TYPE_NOT_STORED);
customType2.StoreTermVectors = true;
customType2.StoreTermVectorPositions = true;
customType2.StoreTermVectorOffsets = true;
Field termVectorField = NewField("termVector", "termVector", customType2);
document.Add(termVectorField);
writer.AddDocument(document);
writer.ForceMerge(1);
writer.Dispose();
IndexReader reader = DirectoryReader.Open(dir);
for (int i = 0; i < reader.NumDocs; i++)
{
reader.Document(i);
reader.GetTermVectors(i);
}
reader.Dispose();
writer = new IndexWriter(dir, ((IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)).SetMergeScheduler(new SerialMergeScheduler()).SetMergePolicy(new LogDocMergePolicy()));
Directory[] indexDirs = new Directory[] { new MockDirectoryWrapper(Random(), new RAMDirectory(dir, NewIOContext(Random()))) };
writer.AddIndexes(indexDirs);
writer.ForceMerge(1);
writer.Dispose();
}
dir.Dispose();
}
public virtual void TestAddIndexes()
{
Directory dir1 = NewDirectory();
Directory dir2 = NewDirectory();
IndexWriter writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.COMPOUND_FILES));
Document d1 = new Document();
d1.Add(new TextField("f1", "first field", Field.Store.YES));
d1.Add(new TextField("f2", "second field", Field.Store.YES));
writer.AddDocument(d1);
writer.Dispose();
writer = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.COMPOUND_FILES));
Document d2 = new Document();
FieldType customType2 = new FieldType(TextField.TYPE_STORED);
customType2.StoreTermVectors = true;
d2.Add(new TextField("f2", "second field", Field.Store.YES));
d2.Add(new Field("f1", "first field", customType2));
d2.Add(new TextField("f3", "third field", Field.Store.YES));
d2.Add(new TextField("f4", "fourth field", Field.Store.YES));
writer.AddDocument(d2);
writer.Dispose();
writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.COMPOUND_FILES));
writer.AddIndexes(dir2);
writer.Dispose();
SegmentInfos sis = new SegmentInfos();
sis.Read(dir1);
Assert.AreEqual(2, sis.Size());
FieldInfos fis1 = SegmentReader.ReadFieldInfos(sis.Info(0));
FieldInfos fis2 = SegmentReader.ReadFieldInfos(sis.Info(1));
Assert.AreEqual("f1", fis1.FieldInfo(0).Name);
Assert.AreEqual("f2", fis1.FieldInfo(1).Name);
// make sure the ordering of the "external" segment is preserved
Assert.AreEqual("f2", fis2.FieldInfo(0).Name);
Assert.AreEqual("f1", fis2.FieldInfo(1).Name);
Assert.AreEqual("f3", fis2.FieldInfo(2).Name);
Assert.AreEqual("f4", fis2.FieldInfo(3).Name);
dir1.Dispose();
dir2.Dispose();
}
public override void Act()
{
var indexDirectory = Path.Combine(CoreConfiguration.DataDirectory, "MergedIndexes");
var directoryInfo = new DirectoryInfo(indexDirectory);
if(directoryInfo.Exists)
{
directoryInfo.Delete(true);
directoryInfo.Refresh();
}
var mergedDirectory = new SimpleFSDirectory(directoryInfo);
var mergedIndex = new IndexWriter(mergedDirectory, new SimpleAnalyzer(), true,
IndexWriter.MaxFieldLength.UNLIMITED);
var directoryFactory = AutofacContainer.Resolve<IDirectoryFactory>();
mergedIndex.AddIndexes(directoryFactory.GetAllDirectories().Select(d => IndexReader.Open(d, true)).ToArray());
mergedIndex.Commit();
}
public static void Syncronize(DirectoryInfo sessionDirectory, IDocumentFileNameProvider fileNameProvider)
{
using (Mutex)
{
using (var fsDirectory = FSDirectory.Open(fileNameProvider.GetLuceneDirectory()))
{
using (var writer = new IndexWriter(fsDirectory, Analyser, IndexWriter.MaxFieldLength.UNLIMITED))
{
using (var readerDirectory = FSDirectory.Open(sessionDirectory))
{
using (var reader = IndexReader.Open(readerDirectory, true))
{
writer.AddIndexes(reader);
}
}
}
}
}
}
public virtual void TestAddOldIndexes()
{
foreach (string name in OldNames)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: old index " + name);
}
Directory targetDir = NewDirectory();
IndexWriter w = new IndexWriter(targetDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
w.AddIndexes(OldIndexDirs[name]);
if (VERBOSE)
{
Console.WriteLine("\nTEST: done adding indices; now close");
}
w.Dispose();
targetDir.Dispose();
}
}
public virtual void TestLucene()
{
int num = 100;
Directory indexA = NewDirectory();
Directory indexB = NewDirectory();
FillIndex(Random, indexA, 0, num);
bool fail = VerifyIndex(indexA, 0);
if (fail)
{
Assert.Fail("Index a is invalid");
}
FillIndex(Random, indexB, num, num);
fail = VerifyIndex(indexB, num);
if (fail)
{
Assert.Fail("Index b is invalid");
}
Directory merged = NewDirectory();
IndexWriter writer = new IndexWriter(merged, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NewLogMergePolicy(2)));
writer.AddIndexes(indexA, indexB);
writer.ForceMerge(1);
writer.Dispose();
fail = VerifyIndex(merged, 0);
Assert.IsFalse(fail, "The merged index is invalid");
indexA.Dispose();
indexB.Dispose();
merged.Dispose();
}
public virtual void TestLucene()
{
int num = 100;
Directory indexA = new MockRAMDirectory();
Directory indexB = new MockRAMDirectory();
FillIndex(indexA, 0, num);
bool fail = VerifyIndex(indexA, 0);
if (fail)
{
Assert.Fail("Index a is invalid");
}
FillIndex(indexB, num, num);
fail = VerifyIndex(indexB, num);
if (fail)
{
Assert.Fail("Index b is invalid");
}
Directory merged = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(merged, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMergeFactor(2);
writer.AddIndexes(new Directory[] { indexA, indexB });
writer.Close();
fail = VerifyIndex(merged, 0);
merged.Close();
Assert.IsFalse(fail, "The merged index is invalid");
}
public virtual void TestAddIndexOnDiskFull()
{
// MemoryCodec, since it uses FST, is not necessarily
// "additive", ie if you add up N small FSTs, then merge
// them, the merged result can easily be larger than the
// sum because the merged FST may use array encoding for
// some arcs (which uses more space):
string idFormat = TestUtil.GetPostingsFormat("id");
string contentFormat = TestUtil.GetPostingsFormat("content");
AssumeFalse("this test cannot run with Memory codec", idFormat.Equals("Memory") || contentFormat.Equals("Memory"));
int START_COUNT = 57;
int NUM_DIR = TEST_NIGHTLY ? 50 : 5;
int END_COUNT = START_COUNT + NUM_DIR * (TEST_NIGHTLY ? 25 : 5);
// Build up a bunch of dirs that have indexes which we
// will then merge together by calling addIndexes(*):
Directory[] dirs = new Directory[NUM_DIR];
long inputDiskUsage = 0;
for (int i = 0; i < NUM_DIR; i++)
{
dirs[i] = NewDirectory();
IndexWriter writer = new IndexWriter(dirs[i], NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
for (int j = 0; j < 25; j++)
{
AddDocWithIndex(writer, 25 * i + j);
}
writer.Dispose();
string[] files = dirs[i].ListAll();
for (int j = 0; j < files.Length; j++)
{
inputDiskUsage += dirs[i].FileLength(files[j]);
}
}
// Now, build a starting index that has START_COUNT docs. We
// will then try to addIndexes into a copy of this:
MockDirectoryWrapper startDir = NewMockDirectory();
IndexWriter indWriter = new IndexWriter(startDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
for (int j = 0; j < START_COUNT; j++)
{
AddDocWithIndex(indWriter, j);
}
indWriter.Dispose();
// Make sure starting index seems to be working properly:
Term searchTerm = new Term("content", "aaa");
IndexReader reader = DirectoryReader.Open(startDir);
Assert.AreEqual(57, reader.DocFreq(searchTerm), "first docFreq");
IndexSearcher searcher = NewSearcher(reader);
ScoreDoc[] hits = searcher.Search(new TermQuery(searchTerm), null, 1000).ScoreDocs;
Assert.AreEqual(57, hits.Length, "first number of hits");
reader.Dispose();
// Iterate with larger and larger amounts of free
// disk space. With little free disk space,
// addIndexes will certainly run out of space &
// fail. Verify that when this happens, index is
// not corrupt and index in fact has added no
// documents. Then, we increase disk space by 2000
// bytes each iteration. At some point there is
// enough free disk space and addIndexes should
// succeed and index should show all documents were
// added.
// String[] files = startDir.ListAll();
long diskUsage = startDir.GetSizeInBytes();
long startDiskUsage = 0;
string[] files_ = startDir.ListAll();
for (int i = 0; i < files_.Length; i++)
{
startDiskUsage += startDir.FileLength(files_[i]);
}
for (int iter = 0; iter < 3; iter++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter);
}
// Start with 100 bytes more than we are currently using:
long diskFree = diskUsage + TestUtil.NextInt(Random(), 50, 200);
int method = iter;
bool success = false;
bool done = false;
string methodName;
if (0 == method)
{
methodName = "addIndexes(Directory[]) + forceMerge(1)";
}
else if (1 == method)
{
methodName = "addIndexes(IndexReader[])";
}
else
{
methodName = "addIndexes(Directory[])";
}
while (!done)
{
if (VERBOSE)
{
Console.WriteLine("TEST: cycle...");
}
// Make a new dir that will enforce disk usage:
MockDirectoryWrapper dir = new MockDirectoryWrapper(Random(), new RAMDirectory(startDir, NewIOContext(Random())));
indWriter = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.APPEND).SetMergePolicy(NewLogMergePolicy(false)));
IOException err = null;
IMergeScheduler ms = indWriter.Config.MergeScheduler;
for (int x = 0; x < 2; x++)
{
if (ms is IConcurrentMergeScheduler)
// this test intentionally produces exceptions
// in the threads that CMS launches; we don't
// want to pollute test output with these.
{
if (0 == x)
{
((IConcurrentMergeScheduler)ms).SetSuppressExceptions();
}
else
{
((IConcurrentMergeScheduler)ms).ClearSuppressExceptions();
}
}
// Two loops: first time, limit disk space &
// throw random IOExceptions; second time, no
// disk space limit:
double rate = 0.05;
double diskRatio = ((double)diskFree) / diskUsage;
long thisDiskFree;
string testName = null;
if (0 == x)
{
dir.RandomIOExceptionRateOnOpen = Random().NextDouble() * 0.01;
thisDiskFree = diskFree;
if (diskRatio >= 2.0)
{
rate /= 2;
}
if (diskRatio >= 4.0)
{
rate /= 2;
}
if (diskRatio >= 6.0)
{
rate = 0.0;
}
if (VERBOSE)
{
testName = "disk full test " + methodName + " with disk full at " + diskFree + " bytes";
}
}
else
{
dir.RandomIOExceptionRateOnOpen = 0.0;
thisDiskFree = 0;
rate = 0.0;
if (VERBOSE)
{
testName = "disk full test " + methodName + " with unlimited disk space";
}
}
if (VERBOSE)
{
Console.WriteLine("\ncycle: " + testName);
}
dir.TrackDiskUsage = true;
dir.MaxSizeInBytes = thisDiskFree;
dir.RandomIOExceptionRate = rate;
try
{
if (0 == method)
{
if (VERBOSE)
{
Console.WriteLine("TEST: now addIndexes count=" + dirs.Length);
}
indWriter.AddIndexes(dirs);
if (VERBOSE)
{
Console.WriteLine("TEST: now forceMerge");
}
indWriter.ForceMerge(1);
}
else if (1 == method)
{
IndexReader[] readers = new IndexReader[dirs.Length];
for (int i = 0; i < dirs.Length; i++)
{
readers[i] = DirectoryReader.Open(dirs[i]);
}
try
{
indWriter.AddIndexes(readers);
}
finally
{
for (int i = 0; i < dirs.Length; i++)
{
readers[i].Dispose();
}
}
}
else
{
indWriter.AddIndexes(dirs);
}
success = true;
if (VERBOSE)
{
Console.WriteLine(" success!");
}
if (0 == x)
{
done = true;
}
}
catch (IOException e)
{
success = false;
err = e;
if (VERBOSE)
{
Console.WriteLine(" hit IOException: " + e);
Console.WriteLine(e.StackTrace);
}
if (1 == x)
{
Console.WriteLine(e.StackTrace);
Assert.Fail(methodName + " hit IOException after disk space was freed up");
}
}
// Make sure all threads from
// ConcurrentMergeScheduler are done
TestUtil.SyncConcurrentMerges(indWriter);
if (VERBOSE)
{
Console.WriteLine(" now test readers");
}
// Finally, verify index is not corrupt, and, if
// we succeeded, we see all docs added, and if we
// failed, we see either all docs or no docs added
// (transactional semantics):
dir.RandomIOExceptionRateOnOpen = 0.0;
try
{
reader = DirectoryReader.Open(dir);
}
catch (IOException e)
{
Console.WriteLine(e.StackTrace);
Assert.Fail(testName + ": exception when creating IndexReader: " + e);
}
int result = reader.DocFreq(searchTerm);
if (success)
{
if (result != START_COUNT)
{
Assert.Fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT);
}
}
else
{
// On hitting exception we still may have added
// all docs:
if (result != START_COUNT && result != END_COUNT)
{
Console.WriteLine(err.StackTrace);
Assert.Fail(testName + ": method did throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
searcher = NewSearcher(reader);
try
{
hits = searcher.Search(new TermQuery(searchTerm), null, END_COUNT).ScoreDocs;
}
catch (IOException e)
{
Console.WriteLine(e.StackTrace);
Assert.Fail(testName + ": exception when searching: " + e);
}
int result2 = hits.Length;
if (success)
{
if (result2 != result)
{
Assert.Fail(testName + ": method did not throw exception but hits.Length for search on term 'aaa' is " + result2 + " instead of expected " + result);
}
}
else
{
// On hitting exception we still may have added
// all docs:
if (result2 != result)
{
Console.WriteLine(err.StackTrace);
Assert.Fail(testName + ": method did throw exception but hits.Length for search on term 'aaa' is " + result2 + " instead of expected " + result);
}
}
reader.Dispose();
if (VERBOSE)
{
Console.WriteLine(" count is " + result);
}
if (done || result == END_COUNT)
{
break;
}
}
if (VERBOSE)
{
Console.WriteLine(" start disk = " + startDiskUsage + "; input disk = " + inputDiskUsage + "; max used = " + dir.MaxUsedSizeInBytes);
}
if (done)
{
// Javadocs state that temp free Directory space
// required is at most 2X total input size of
// indices so let's make sure:
Assert.IsTrue((dir.MaxUsedSizeInBytes - startDiskUsage) < 2 * (startDiskUsage + inputDiskUsage), "max free Directory space required exceeded 1X the total input index sizes during " + methodName + ": max temp usage = " + (dir.MaxUsedSizeInBytes - startDiskUsage) + " bytes vs limit=" + (2 * (startDiskUsage + inputDiskUsage)) + "; starting disk usage = " + startDiskUsage + " bytes; " + "input index disk usage = " + inputDiskUsage + " bytes");
}
// Make sure we don't hit disk full during close below:
dir.MaxSizeInBytes = 0;
dir.RandomIOExceptionRate = 0.0;
dir.RandomIOExceptionRateOnOpen = 0.0;
indWriter.Dispose();
// Wait for all BG threads to finish else
// dir.Dispose() will throw IOException because
// there are still open files
TestUtil.SyncConcurrentMerges(ms);
dir.Dispose();
// Try again with more free space:
diskFree += TEST_NIGHTLY ? TestUtil.NextInt(Random(), 4000, 8000) : TestUtil.NextInt(Random(), 40000, 80000);
}
}
startDir.Dispose();
foreach (Directory dir in dirs)
{
dir.Dispose();
}
}
public virtual void TestNonCFSLeftovers()
{
Directory[] dirs = new Directory[2];
for (int i = 0; i < dirs.Length; i++)
{
dirs[i] = new RAMDirectory();
IndexWriter w = new IndexWriter(dirs[i], new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document d = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
d.Add(new Field("c", "v", customType));
w.AddDocument(d);
w.Dispose();
}
IndexReader[] readers = new IndexReader[] { DirectoryReader.Open(dirs[0]), DirectoryReader.Open(dirs[1]) };
Directory dir = new MockDirectoryWrapper(Random(), new RAMDirectory());
IndexWriterConfig conf = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMergePolicy(NewLogMergePolicy(true));
MergePolicy lmp = conf.MergePolicy;
// Force creation of CFS:
lmp.NoCFSRatio = 1.0;
lmp.MaxCFSSegmentSizeMB = double.PositiveInfinity;
IndexWriter w3 = new IndexWriter(dir, conf);
w3.AddIndexes(readers);
w3.Dispose();
// we should now see segments_X,
// segments.gen,_Y.cfs,_Y.cfe, _Z.si
Assert.AreEqual(5, dir.ListAll().Length, "Only one compound segment should exist, but got: " + Arrays.ToString(dir.ListAll()));
dir.Dispose();
}
public virtual void TestTypeChangeViaAddIndexesIR2()
{
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)conf.Clone());
Document doc = new Document();
doc.Add(new NumericDocValuesField("dv", 0L));
writer.AddDocument(doc);
writer.Dispose();
Directory dir2 = NewDirectory();
writer = new IndexWriter(dir2, (IndexWriterConfig)conf.Clone());
IndexReader[] readers = new IndexReader[] { DirectoryReader.Open(dir) };
writer.AddIndexes(readers);
readers[0].Dispose();
doc = new Document();
doc.Add(new SortedDocValuesField("dv", new BytesRef("foo")));
try
{
writer.AddDocument(doc);
Assert.Fail("did not hit exception");
}
catch (System.ArgumentException iae)
{
// expected
}
writer.Dispose();
dir2.Dispose();
dir.Dispose();
}
public virtual void Test()
{
MockDirectoryWrapper dir = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors"));
dir.PreventDoubleWrite = false;
double rate = Random.NextDouble() * 0.01;
//System.out.println("rate=" + rate);
dir.RandomIOExceptionRateOnOpen = rate;
int iters = AtLeast(20);
LineFileDocs docs = new LineFileDocs(Random, DefaultCodecSupportsDocValues);
IndexReader r = null;
DirectoryReader r2 = null;
bool any = false;
MockDirectoryWrapper dirCopy = null;
int lastNumDocs = 0;
for (int iter = 0; iter < iters; iter++)
{
IndexWriter w = null;
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter);
}
try
{
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
if (VERBOSE)
{
// Do this ourselves instead of relying on LTC so
// we see incrementing messageID:
iwc.SetInfoStream(new TextWriterInfoStream(Console.Out));
}
var ms = iwc.MergeScheduler;
if (ms is IConcurrentMergeScheduler)
{
((IConcurrentMergeScheduler)ms).SetSuppressExceptions();
}
w = new IndexWriter(dir, iwc);
if (r != null && Random.Next(5) == 3)
{
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: addIndexes IR[]");
}
w.AddIndexes(new IndexReader[] { r });
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: addIndexes Directory[]");
}
w.AddIndexes(new Directory[] { dirCopy });
}
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: addDocument");
}
w.AddDocument(docs.NextDoc());
}
dir.RandomIOExceptionRateOnOpen = 0.0;
w.Dispose();
w = null;
// NOTE: this is O(N^2)! Only enable for temporary debugging:
//dir.setRandomIOExceptionRateOnOpen(0.0);
//TestUtil.CheckIndex(dir);
//dir.setRandomIOExceptionRateOnOpen(rate);
// Verify numDocs only increases, to catch IndexWriter
// accidentally deleting the index:
dir.RandomIOExceptionRateOnOpen = 0.0;
Assert.IsTrue(DirectoryReader.IndexExists(dir));
if (r2 == null)
{
r2 = DirectoryReader.Open(dir);
}
else
{
DirectoryReader r3 = DirectoryReader.OpenIfChanged(r2);
if (r3 != null)
{
r2.Dispose();
r2 = r3;
}
}
Assert.IsTrue(r2.NumDocs >= lastNumDocs, "before=" + lastNumDocs + " after=" + r2.NumDocs);
lastNumDocs = r2.NumDocs;
//System.out.println("numDocs=" + lastNumDocs);
dir.RandomIOExceptionRateOnOpen = rate;
any = true;
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + ": success");
}
}
catch (IOException ioe)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + ": exception");
Console.WriteLine(ioe.ToString());
Console.Write(ioe.StackTrace);
}
if (w != null)
{
// NOTE: leave random IO exceptions enabled here,
// to verify that rollback does not try to write
// anything:
w.Rollback();
}
}
if (any && r == null && Random.NextBoolean())
{
// Make a copy of a non-empty index so we can use
// it to addIndexes later:
dir.RandomIOExceptionRateOnOpen = 0.0;
r = DirectoryReader.Open(dir);
dirCopy = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors.copy"));
HashSet <string> files = new HashSet <string>();
foreach (string file in dir.ListAll())
{
dir.Copy(dirCopy, file, file, IOContext.DEFAULT);
files.Add(file);
}
dirCopy.Sync(files);
// Have IW kiss the dir so we remove any leftover
// files ... we can easily have leftover files at
// the time we take a copy because we are holding
// open a reader:
(new IndexWriter(dirCopy, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)))).Dispose();
dirCopy.RandomIOExceptionRate = rate;
dir.RandomIOExceptionRateOnOpen = rate;
}
}
if (r2 != null)
{
r2.Dispose();
}
if (r != null)
{
r.Dispose();
dirCopy.Dispose();
}
dir.Dispose();
}
public virtual void Test([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BPostingsBytes1"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
var config = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.SetRAMBufferSizeMB(256.0)
.SetMergeScheduler(scheduler)
.SetMergePolicy(NewLogMergePolicy(false, 10))
.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE);
IndexWriter w = new IndexWriter(dir, config);
MergePolicy mp = w.Config.MergePolicy;
if (mp is LogByteSizeMergePolicy)
{
// 1 petabyte:
((LogByteSizeMergePolicy)mp).MaxMergeMB = 1024 * 1024 * 1024;
}
Document doc = new Document();
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS;
ft.OmitNorms = true;
MyTokenStream tokenStream = new MyTokenStream();
Field field = new Field("field", tokenStream, ft);
doc.Add(field);
const int numDocs = 1000;
for (int i = 0; i < numDocs; i++)
{
if (i % 2 == 1) // trick blockPF's little optimization
{
tokenStream.n = 65536;
}
else
{
tokenStream.n = 65537;
}
w.AddDocument(doc);
}
w.ForceMerge(1);
w.Dispose();
DirectoryReader oneThousand = DirectoryReader.Open(dir);
IndexReader[] subReaders = new IndexReader[1000];
Arrays.Fill(subReaders, oneThousand);
MultiReader mr = new MultiReader(subReaders);
BaseDirectoryWrapper dir2 = NewFSDirectory(CreateTempDir("2BPostingsBytes2"));
if (dir2 is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir2).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
IndexWriter w2 = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
w2.AddIndexes(mr);
w2.ForceMerge(1);
w2.Dispose();
oneThousand.Dispose();
DirectoryReader oneMillion = DirectoryReader.Open(dir2);
subReaders = new IndexReader[2000];
Arrays.Fill(subReaders, oneMillion);
mr = new MultiReader(subReaders);
BaseDirectoryWrapper dir3 = NewFSDirectory(CreateTempDir("2BPostingsBytes3"));
if (dir3 is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir3).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
IndexWriter w3 = new IndexWriter(dir3, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
w3.AddIndexes(mr);
w3.ForceMerge(1);
w3.Dispose();
oneMillion.Dispose();
dir.Dispose();
dir2.Dispose();
dir3.Dispose();
}
public virtual void TestAddIndexes()
{
Directory dir1 = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter writer = new IndexWriter(dir1, conf);
int numDocs = AtLeast(50);
int numTerms = TestUtil.NextInt(Random(), 1, numDocs / 5);
HashSet<string> randomTerms = new HashSet<string>();
while (randomTerms.Count < numTerms)
{
randomTerms.Add(TestUtil.RandomSimpleString(Random()));
}
// create first index
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
doc.Add(new StringField("id", RandomInts.RandomFrom(Random(), randomTerms), Store.NO));
doc.Add(new NumericDocValuesField("ndv", 4L));
doc.Add(new NumericDocValuesField("control", 8L));
writer.AddDocument(doc);
}
if (Random().NextBoolean())
{
writer.Commit();
}
// update some docs to a random value
long value = Random().Next();
Term term = new Term("id", RandomInts.RandomFrom(Random(), randomTerms));
writer.UpdateNumericDocValue(term, "ndv", value);
writer.UpdateNumericDocValue(term, "control", value * 2);
writer.Dispose();
Directory dir2 = NewDirectory();
conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
writer = new IndexWriter(dir2, conf);
if (Random().NextBoolean())
{
writer.AddIndexes(dir1);
}
else
{
DirectoryReader reader = DirectoryReader.Open(dir1);
writer.AddIndexes(reader);
reader.Dispose();
}
writer.Dispose();
DirectoryReader reader_ = DirectoryReader.Open(dir2);
foreach (AtomicReaderContext context in reader_.Leaves)
{
AtomicReader r = context.AtomicReader;
NumericDocValues ndv = r.GetNumericDocValues("ndv");
NumericDocValues control = r.GetNumericDocValues("control");
for (int i = 0; i < r.MaxDoc; i++)
{
Assert.AreEqual(ndv.Get(i) * 2, control.Get(i));
}
}
reader_.Dispose();
IOUtils.Close(dir1, dir2);
}
/// <summary>
/// Split source index into multiple parts. </summary>
/// <param name="version">lucene compatibility version</param>
/// <param name="in"> source index, can have deletions, can have
/// multiple segments (or multiple readers). </param>
/// <param name="outputs"> list of directories where the output parts will be stored. </param>
/// <param name="seq"> if true, then the source index will be split into equal
/// increasing ranges of document id-s. If false, source document id-s will be
/// assigned in a deterministic round-robin fashion to one of the output splits. </param>
/// <exception cref="IOException"> If there is a low-level I/O error </exception>
public virtual void Split(LuceneVersion version, IndexReader @in, Store.Directory[] outputs, bool seq)
{
if (outputs == null || outputs.Length < 2)
{
throw new IOException("Invalid number of outputs.");
}
if (@in == null || @in.NumDocs < 2)
{
throw new IOException("Not enough documents for splitting");
}
int numParts = outputs.Length;
// wrap a potentially read-only input
// this way we don't have to preserve original deletions because neither
// deleteDocument(int) or undeleteAll() is applied to the wrapped input index.
FakeDeleteIndexReader input = new FakeDeleteIndexReader(@in);
int maxDoc = input.MaxDoc;
int partLen = maxDoc / numParts;
for (int i = 0; i < numParts; i++)
{
input.UndeleteAll();
if (seq) // sequential range
{
int lo = partLen * i;
int hi = lo + partLen;
// below range
for (int j = 0; j < lo; j++)
{
input.DeleteDocument(j);
}
// above range - last part collects all id-s that remained due to
// integer rounding errors
if (i < numParts - 1)
{
for (int j = hi; j < maxDoc; j++)
{
input.DeleteDocument(j);
}
}
}
else
{
// round-robin
for (int j = 0; j < maxDoc; j++)
{
if ((j + numParts - i) % numParts != 0)
{
input.DeleteDocument(j);
}
}
}
using (IndexWriter w = new IndexWriter(outputs[i],
new IndexWriterConfig(version, null)
{
OpenMode = OpenMode.CREATE
}))
{
Console.Error.WriteLine("Writing part " + (i + 1) + " ...");
// pass the subreaders directly, as our wrapper's numDocs/hasDeletetions are not up-to-date
IList <IndexReader> sr = input.GetSequentialSubReaders();
w.AddIndexes(sr.ToArray()); // TODO: maybe take List<IR> here?
}
}
Console.Error.WriteLine("Done.");
}
public virtual void TestAddIndexes2()
{
bool doFullMerge = false;
Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
IndexWriter writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
// create a 2nd index
Directory dir2 = NewDirectory();
IndexWriter writer2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
CreateIndexNoClose(!doFullMerge, "index2", writer2);
writer2.Dispose();
writer.AddIndexes(dir2);
writer.AddIndexes(dir2);
writer.AddIndexes(dir2);
writer.AddIndexes(dir2);
writer.AddIndexes(dir2);
IndexReader r1 = writer.Reader;
Assert.AreEqual(500, r1.MaxDoc());
r1.Dispose();
writer.Dispose();
dir1.Dispose();
dir2.Dispose();
}
public virtual void TestMergeAfterCopy()
{
// main directory
Directory dir = NewDirectory();
// auxiliary directory
Directory aux = NewDirectory();
SetUpDirs(dir, aux, true);
IndexWriterConfig dontMergeConfig = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
IndexWriter writer = new IndexWriter(aux, dontMergeConfig);
for (int i = 0; i < 20; i++)
{
writer.DeleteDocuments(new Term("id", "" + i));
}
writer.Dispose();
IndexReader reader = DirectoryReader.Open(aux);
Assert.AreEqual(10, reader.NumDocs);
reader.Dispose();
writer = NewWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.APPEND).SetMaxBufferedDocs(4).SetMergePolicy(NewLogMergePolicy(4)));
if (VERBOSE)
{
Console.WriteLine("\nTEST: now addIndexes");
}
writer.AddIndexes(aux, new MockDirectoryWrapper(Random(), new RAMDirectory(aux, NewIOContext(Random()))));
Assert.AreEqual(1020, writer.MaxDoc);
Assert.AreEqual(1000, writer.GetDocCount(0));
writer.Dispose();
dir.Dispose();
aux.Dispose();
}
public virtual void TestAddIndexMissingCodec()
{
BaseDirectoryWrapper toAdd = NewDirectory();
// Disable checkIndex, else we get an exception because
// of the unregistered codec:
toAdd.CheckIndexOnClose = false;
{
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
conf.SetCodec(new UnRegisteredCodec());
using (var w = new IndexWriter(toAdd, conf))
{
Document doc = new Document();
FieldType customType = new FieldType();
customType.Indexed = true;
doc.Add(NewField("foo", "bar", customType));
w.AddDocument(doc);
}
}
{
using (Directory dir = NewDirectory())
{
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
conf.SetCodec(TestUtil.AlwaysPostingsFormat(new Pulsing41PostingsFormat(1 + Random().Next(20))));
IndexWriter w = new IndexWriter(dir, conf);
try
{
w.AddIndexes(toAdd);
Assert.Fail("no such codec");
}
catch (System.ArgumentException ex)
{
// expected
}
finally
{
w.Dispose();
}
using (IndexReader open = DirectoryReader.Open(dir))
{
Assert.AreEqual(0, open.NumDocs);
}
}
}
try
{
DirectoryReader.Open(toAdd);
Assert.Fail("no such codec");
}
catch (System.ArgumentException ex)
{
// expected
}
toAdd.Dispose();
}
public virtual void AddIndexes(params Directory[] dirs) => IndexWriter.AddIndexes(dirs);
public virtual void TestUpgradeOldSingleSegmentIndexWithAdditions()
{
foreach (string name in OldSingleSegmentNames)
{
if (VERBOSE)
{
Console.WriteLine("testUpgradeOldSingleSegmentIndexWithAdditions: index=" + name);
}
Directory dir = NewDirectory(OldIndexDirs[name]);
Assert.AreEqual(1, GetNumberOfSegments(dir), "Original index must be single segment");
// create a bunch of dummy segments
int id = 40;
RAMDirectory ramDir = new RAMDirectory();
for (int i = 0; i < 3; i++)
{
// only use Log- or TieredMergePolicy, to make document addition predictable and not suddenly merge:
MergePolicy mp = Random().NextBoolean() ? (MergePolicy)NewLogMergePolicy() : NewTieredMergePolicy();
IndexWriterConfig iwc = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMergePolicy(mp);
IndexWriter w = new IndexWriter(ramDir, iwc);
// add few more docs:
for (int j = 0; j < RANDOM_MULTIPLIER * Random().Next(30); j++)
{
AddDoc(w, id++);
}
w.Dispose(false);
}
// add dummy segments (which are all in current
// version) to single segment index
MergePolicy mp_ = Random().NextBoolean() ? (MergePolicy)NewLogMergePolicy() : NewTieredMergePolicy();
IndexWriterConfig iwc_ = (new IndexWriterConfig(TEST_VERSION_CURRENT, null)).SetMergePolicy(mp_);
IndexWriter iw = new IndexWriter(dir, iwc_);
iw.AddIndexes(ramDir);
iw.Dispose(false);
// determine count of segments in modified index
int origSegCount = GetNumberOfSegments(dir);
NewIndexUpgrader(dir).Upgrade();
int segCount = CheckAllSegmentsUpgraded(dir);
Assert.AreEqual(origSegCount, segCount, "Index must still contain the same number of segments, as only one segment was upgraded and nothing else merged");
dir.Dispose();
}
}
public virtual void TestTermVectorCorruption()
{
Directory dir = new MockRAMDirectory();
for (int iter = 0; iter < 4; iter++)
{
bool autoCommit = 1 == iter / 2;
IndexWriter writer = new IndexWriter(dir, autoCommit, new StandardAnalyzer());
writer.SetMaxBufferedDocs(2);
writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
writer.SetMergeScheduler(new SerialMergeScheduler());
writer.SetMergePolicy(new LogDocMergePolicy());
Document document = new Document();
Field storedField = new Field("stored", "stored", Field.Store.YES, Field.Index.NO);
document.Add(storedField);
writer.AddDocument(document);
writer.AddDocument(document);
document = new Document();
document.Add(storedField);
Field termVectorField = new Field("termVector", "termVector", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
document.Add(termVectorField);
writer.AddDocument(document);
writer.Optimize();
writer.Close();
IndexReader reader = IndexReader.Open(dir);
for (int i = 0; i < reader.NumDocs(); i++)
{
reader.Document(i);
reader.GetTermFreqVectors(i);
}
reader.Close();
writer = new IndexWriter(dir, autoCommit, new StandardAnalyzer());
writer.SetMaxBufferedDocs(2);
writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
writer.SetMergeScheduler(new SerialMergeScheduler());
writer.SetMergePolicy(new LogDocMergePolicy());
Directory[] indexDirs = new Directory[]{dir};
writer.AddIndexes(indexDirs);
writer.Close();
}
dir.Close();
}
public virtual void TestAddOldIndexesReader()
{
foreach (string name in OldNames)
{
IndexReader reader = DirectoryReader.Open(OldIndexDirs[name]);
Directory targetDir = NewDirectory();
IndexWriter w = new IndexWriter(targetDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
w.AddIndexes(reader);
w.Dispose();
reader.Dispose();
targetDir.Dispose();
}
}
public virtual void TestNorms()
{
// tmp dir
System.String tempDir = System.IO.Path.GetTempPath();
if (tempDir == null)
{
throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test");
}
// test with a single index: index1
System.IO.FileInfo indexDir1 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex1"));
Directory dir1 = FSDirectory.Open(indexDir1);
IndexWriter.Unlock(dir1);
norms = new System.Collections.ArrayList();
modifiedNorms = new System.Collections.ArrayList();
CreateIndex(dir1);
DoTestNorms(dir1);
// test with a single index: index2
System.Collections.ArrayList norms1 = norms;
System.Collections.ArrayList modifiedNorms1 = modifiedNorms;
int numDocNorms1 = numDocNorms;
norms = new System.Collections.ArrayList();
modifiedNorms = new System.Collections.ArrayList();
numDocNorms = 0;
System.IO.FileInfo indexDir2 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex2"));
Directory dir2 = FSDirectory.Open(indexDir2);
CreateIndex(dir2);
DoTestNorms(dir2);
// add index1 and index2 to a third index: index3
System.IO.FileInfo indexDir3 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex3"));
Directory dir3 = FSDirectory.Open(indexDir3);
CreateIndex(dir3);
IndexWriter iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.AddIndexes(new Directory[] { dir1, dir2 });
iw.Close();
norms1.AddRange(norms);
norms = norms1;
modifiedNorms1.AddRange(modifiedNorms);
modifiedNorms = modifiedNorms1;
numDocNorms += numDocNorms1;
// test with index3
VerifyIndex(dir3);
DoTestNorms(dir3);
// now with optimize
iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.Optimize();
iw.Close();
VerifyIndex(dir3);
dir1.Close();
dir2.Close();
dir3.Close();
}
public virtual void TestAddIndexOnDiskFull()
{
int START_COUNT = 57;
int NUM_DIR = 50;
int END_COUNT = START_COUNT + NUM_DIR * 25;
bool debug = false;
// Build up a bunch of dirs that have indexes which we
// will then merge together by calling addIndexes(*):
Directory[] dirs = new Directory[NUM_DIR];
long inputDiskUsage = 0;
for (int i = 0; i < NUM_DIR; i++)
{
dirs[i] = new RAMDirectory();
IndexWriter writer = new IndexWriter(dirs[i], new WhitespaceAnalyzer(), true);
for (int j = 0; j < 25; j++)
{
AddDocWithIndex(writer, 25 * i + j);
}
writer.Close();
System.String[] files = dirs[i].List();
for (int j = 0; j < files.Length; j++)
{
inputDiskUsage += dirs[i].FileLength(files[j]);
}
}
// Now, build a starting index that has START_COUNT docs. We
// will then try to addIndexes into a copy of this:
RAMDirectory startDir = new RAMDirectory();
IndexWriter writer2 = new IndexWriter(startDir, new WhitespaceAnalyzer(), true);
for (int j = 0; j < START_COUNT; j++)
{
AddDocWithIndex(writer2, j);
}
writer2.Close();
// Make sure starting index seems to be working properly:
Term searchTerm = new Term("content", "aaa");
IndexReader reader = IndexReader.Open(startDir);
Assert.AreEqual(57, reader.DocFreq(searchTerm), "first docFreq");
IndexSearcher searcher = new IndexSearcher(reader);
Hits hits = searcher.Search(new TermQuery(searchTerm));
Assert.AreEqual(57, hits.Length(), "first number of hits");
searcher.Close();
reader.Close();
// Iterate with larger and larger amounts of free
// disk space. With little free disk space,
// addIndexes will certainly run out of space &
// fail. Verify that when this happens, index is
// not corrupt and index in fact has added no
// documents. Then, we increase disk space by 2000
// bytes each iteration. At some point there is
// enough free disk space and addIndexes should
// succeed and index should show all documents were
// added.
// String[] files = startDir.list();
long diskUsage = startDir.SizeInBytes();
long startDiskUsage = 0;
System.String[] files2 = startDir.List();
for (int i = 0; i < files2.Length; i++)
{
startDiskUsage += startDir.FileLength(files2[i]);
}
for (int iter = 0; iter < 6; iter++)
{
if (debug)
System.Console.Out.WriteLine("TEST: iter=" + iter);
// Start with 100 bytes more than we are currently using:
long diskFree = diskUsage + 100;
bool autoCommit = iter % 2 == 0;
int method = iter / 2;
bool success = false;
bool done = false;
System.String methodName;
if (0 == method)
{
methodName = "addIndexes(Directory[])";
}
else if (1 == method)
{
methodName = "addIndexes(IndexReader[])";
}
else
{
methodName = "addIndexesNoOptimize(Directory[])";
}
while (!done)
{
// Make a new dir that will enforce disk usage:
MockRAMDirectory dir = new MockRAMDirectory(startDir);
writer2 = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
System.IO.IOException err = null;
MergeScheduler ms = writer2.GetMergeScheduler();
for (int x = 0; x < 2; x++)
{
if (ms is ConcurrentMergeScheduler)
// This test intentionally produces exceptions
// in the threads that CMS launches; we don't
// want to pollute test output with these.
if (0 == x)
((ConcurrentMergeScheduler)ms).SetSuppressExceptions_ForNUnitTest();
else
((ConcurrentMergeScheduler) ms).ClearSuppressExceptions_ForNUnitTest();
// Two loops: first time, limit disk space &
// throw random IOExceptions; second time, no
// disk space limit:
double rate = 0.05;
double diskRatio = ((double) diskFree) / diskUsage;
long thisDiskFree;
System.String testName = null;
if (0 == x)
{
thisDiskFree = diskFree;
if (diskRatio >= 2.0)
{
rate /= 2;
}
if (diskRatio >= 4.0)
{
rate /= 2;
}
if (diskRatio >= 6.0)
{
rate = 0.0;
}
if (debug)
testName = "disk full test " + methodName + " with disk full at " + diskFree + " bytes autoCommit=" + autoCommit;
}
else
{
thisDiskFree = 0;
rate = 0.0;
if (debug)
testName = "disk full test " + methodName + " with unlimited disk space autoCommit=" + autoCommit;
}
if (debug)
System.Console.Out.WriteLine("\ncycle: " + testName);
dir.SetMaxSizeInBytes(thisDiskFree);
dir.SetRandomIOExceptionRate(rate, diskFree);
try
{
if (0 == method)
{
writer2.AddIndexes(dirs);
}
else if (1 == method)
{
IndexReader[] readers = new IndexReader[dirs.Length];
for (int i = 0; i < dirs.Length; i++)
{
readers[i] = IndexReader.Open(dirs[i]);
}
try
{
writer2.AddIndexes(readers);
}
finally
{
for (int i = 0; i < dirs.Length; i++)
{
readers[i].Close();
}
}
}
else
{
writer2.AddIndexesNoOptimize(dirs);
}
success = true;
if (debug)
{
System.Console.Out.WriteLine(" success!");
}
if (0 == x)
{
done = true;
}
}
catch (System.IO.IOException e)
{
success = false;
err = e;
if (debug)
{
System.Console.Out.WriteLine(" hit IOException: " + e);
System.Console.Out.WriteLine(e.StackTrace);
}
if (1 == x)
{
System.Console.Out.WriteLine(e.StackTrace);
Assert.Fail(methodName + " hit IOException after disk space was freed up");
}
}
// Make sure all threads from
// ConcurrentMergeScheduler are done
_TestUtil.SyncConcurrentMerges(writer2);
if (autoCommit)
{
// Whether we succeeded or failed, check that
// all un-referenced files were in fact
// deleted (ie, we did not create garbage).
// Only check this when autoCommit is true:
// when it's false, it's expected that there
// are unreferenced files (ie they won't be
// referenced until the "commit on close").
// Just create a new IndexFileDeleter, have it
// delete unreferenced files, then verify that
// in fact no files were deleted:
System.String successStr;
if (success)
{
successStr = "success";
}
else
{
successStr = "IOException";
}
System.String message = methodName + " failed to delete unreferenced files after " + successStr + " (" + diskFree + " bytes)";
AssertNoUnreferencedFiles(dir, message);
}
if (debug)
{
System.Console.Out.WriteLine(" now test readers");
}
// Finally, verify index is not corrupt, and, if
// we succeeded, we see all docs added, and if we
// failed, we see either all docs or no docs added
// (transactional semantics):
try
{
reader = IndexReader.Open(dir);
}
catch (System.IO.IOException e)
{
System.Console.Out.WriteLine(e.StackTrace);
Assert.Fail(testName + ": exception when creating IndexReader: " + e);
}
int result = reader.DocFreq(searchTerm);
if (success)
{
if (autoCommit && result != END_COUNT)
{
Assert.Fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + END_COUNT);
}
else if (!autoCommit && result != START_COUNT)
{
Assert.Fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT + " [autoCommit = false]");
}
}
else
{
// On hitting exception we still may have added
// all docs:
if (result != START_COUNT && result != END_COUNT)
{
System.Console.Out.WriteLine(err.StackTrace);
Assert.Fail(testName + ": method did throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
searcher = new IndexSearcher(reader);
try
{
hits = searcher.Search(new TermQuery(searchTerm));
}
catch (System.IO.IOException e)
{
System.Console.Out.WriteLine(e.StackTrace);
Assert.Fail(testName + ": exception when searching: " + e);
}
int result2 = hits.Length();
if (success)
{
if (result2 != result)
{
Assert.Fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + result);
}
}
else
{
// On hitting exception we still may have added
// all docs:
if (result2 != result)
{
System.Console.Out.WriteLine(err.StackTrace);
Assert.Fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + result);
}
}
searcher.Close();
reader.Close();
if (debug)
{
System.Console.Out.WriteLine(" count is " + result);
}
if (done || result == END_COUNT)
{
break;
}
}
if (debug)
{
System.Console.Out.WriteLine(" start disk = " + startDiskUsage + "; input disk = " + inputDiskUsage + "; max used = " + dir.GetMaxUsedSizeInBytes());
}
if (done)
{
// Javadocs state that temp free Directory space
// required is at most 2X total input size of
// indices so let's make sure:
Assert.IsTrue(
(dir.GetMaxUsedSizeInBytes() - startDiskUsage) < 2 * (startDiskUsage + inputDiskUsage),
"max free Directory space required exceeded 1X the total input index sizes during " + methodName + ": max temp usage = " + (dir.GetMaxUsedSizeInBytes() - startDiskUsage) + " bytes; " + "starting disk usage = " + startDiskUsage + " bytes; " + "input index disk usage = " + inputDiskUsage + " bytes"
);
}
writer2.Close();
// Wait for all BG threads to finish else
// dir.close() will throw IOException because
// there are still open files
_TestUtil.SyncConcurrentMerges(ms);
dir.Close();
// Try again with 2000 more bytes of free space:
diskFree += 2000;
}
}
startDir.Close();
}
public virtual void TestMixedTypesViaAddIndexes()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document doc = new Document();
doc.Add(new NumericDocValuesField("foo", 0));
w.AddDocument(doc);
// Make 2nd index w/ inconsistent field
Directory dir2 = NewDirectory();
IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
doc = new Document();
doc.Add(new SortedDocValuesField("foo", new BytesRef("hello")));
w2.AddDocument(doc);
w2.Dispose();
try
{
w.AddIndexes(dir2);
}
catch (System.ArgumentException iae)
{
// expected
}
IndexReader r = DirectoryReader.Open(dir2);
try
{
w.AddIndexes(new IndexReader[] { r });
}
catch (System.ArgumentException iae)
{
// expected
}
r.Dispose();
dir2.Dispose();
w.Dispose();
dir.Dispose();
}
/// <summary>
/// Creates the index for the bz2 file on a separate thread.
/// </summary>
private void CreateIndexAsync()
{
bool failed = false;
try
{
// Close any searchers
if (searcher != null)
{
searcher.Close();
searcher = null;
}
indexExists = false;
// Create the index writer
indexer = new IndexWriter(indexPath, textAnalyzer, true);
memoryIndexer = new IndexWriter(new RAMDirectory(), textAnalyzer, true);
memoryIndexer.SetMaxBufferedDocs(1000);
memoryIndexer.SetMergeFactor(100);
indexer.SetMaxBufferedDocs(1000);
indexer.SetMergeFactor(100);
// Locate the bzip2 blocks in the file
LocateBlocks();
// Two times more than the first block but not less than 100 bytes
long bufSize = ((ends[0] - beginnings[0]) / 8) * 2 + 100;
// Buffers for the current and next block
blockBuf = new byte[bufSize];
charBuf = new char[bufSize];
// Whether there was a Wiki topic carryover from current block to the next one
char[] charCarryOver = new char[0];
// The length of the currently loaded data
long loadedLength = 0;
StringBuilder sb = new StringBuilder();
// Starting indexing
ReportProgress(0, IndexingProgress.State.Running, "Indexing");
for (long i = 0; i < totalBlocks && !abortIndexing; i++)
{
ReportProgress((int)((double)(i * 100) / (double)totalBlocks), IndexingProgress.State.Running, String.Empty);
#region Indexing logic
loadedLength = LoadBlock(beginnings[i], ends[i], ref blockBuf);
if (charBuf.Length < blockBuf.Length)
{
charBuf = new char[blockBuf.Length];
}
int bytesUsed = 0;
int charsUsed = 0;
bool completed = false;
// Convert the text to UTF8
utf8.Convert(blockBuf, 0, (int)loadedLength, charBuf, 0, charBuf.Length, i == totalBlocks - 1, out bytesUsed, out charsUsed, out completed);
if (!completed)
{
throw new Exception("UTF8 decoder could not complete the conversion");
}
// Construct a current string
sb.Length = 0;
if (charCarryOver.Length > 0)
{
sb.Append(charCarryOver);
}
sb.Append(charBuf, 0, charsUsed);
int carryOverLength = charCarryOver.Length;
int charsMatched = IndexString(sb.ToString(), beginnings[i], ends[i], carryOverLength, i == totalBlocks - 1);
// There's a Wiki topic carryover, let's store the characters which need to be carried over
if (charsMatched > 0)
{
charCarryOver = new char[charsMatched];
sb.CopyTo(charsUsed + carryOverLength - charsMatched, charCarryOver, 0, charsMatched);
}
else
{
charCarryOver = new char[0];
}
#endregion
}
// Wait till all the threads finish
while (activeThreads != 0)
{
ReportProgress(0, IndexingProgress.State.Running, "Waiting for tokenizer threads to finish");
Thread.Sleep(TimeSpan.FromSeconds(5));
}
ReportProgress(0, IndexingProgress.State.Running, "Flushing documents to disk");
Lucene.Net.Store.Directory dir = memoryIndexer.GetDirectory();
memoryIndexer.Close();
indexer.AddIndexes(new Lucene.Net.Store.Directory[] { dir });
memoryIndexer = null;
ReportProgress(0, IndexingProgress.State.Running, "Optimizing index");
indexer.Optimize();
indexExists = true;
}
catch (Exception ex)
{
ReportProgress(0, IndexingProgress.State.Failure, ex.ToString());
failed = true;
}
// Try to release some memory
if (indexer != null)
{
indexer.Close();
indexer = null;
}
if (failed ||
abortIndexing)
{
Directory.Delete(indexPath, true);
indexExists = false;
}
else
{
if (indexExists)
{
searcher = new IndexSearcher(indexPath);
}
}
ReportProgress(0, IndexingProgress.State.Finished, String.Empty);
}
/// <summary>
/// Calls <see cref="IndexWriter.AddIndexes(Directory[])"/> and
/// returns the generation that reflects this change.
/// </summary>
public virtual long AddIndexes(params Directory[] dirs)
{
writer.AddIndexes(dirs);
// Return gen as of when indexing finished:
return(indexingGen.Get());
}
public virtual void Test([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")] IConcurrentMergeScheduler scheduler)
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BPostingsBytes1"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
var config = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.SetRAMBufferSizeMB(256.0)
.SetMergeScheduler(scheduler)
.SetMergePolicy(NewLogMergePolicy(false, 10))
.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE);
IndexWriter w = new IndexWriter(dir, config);
MergePolicy mp = w.Config.MergePolicy;
if (mp is LogByteSizeMergePolicy)
{
// 1 petabyte:
((LogByteSizeMergePolicy)mp).MaxMergeMB = 1024 * 1024 * 1024;
}
Document doc = new Document();
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS;
ft.OmitNorms = true;
MyTokenStream tokenStream = new MyTokenStream();
Field field = new Field("field", tokenStream, ft);
doc.Add(field);
const int numDocs = 1000;
for (int i = 0; i < numDocs; i++)
{
if (i % 2 == 1) // trick blockPF's little optimization
{
tokenStream.n = 65536;
}
else
{
tokenStream.n = 65537;
}
w.AddDocument(doc);
}
w.ForceMerge(1);
w.Dispose();
DirectoryReader oneThousand = DirectoryReader.Open(dir);
IndexReader[] subReaders = new IndexReader[1000];
Arrays.Fill(subReaders, oneThousand);
MultiReader mr = new MultiReader(subReaders);
BaseDirectoryWrapper dir2 = NewFSDirectory(CreateTempDir("2BPostingsBytes2"));
if (dir2 is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir2).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
IndexWriter w2 = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
w2.AddIndexes(mr);
w2.ForceMerge(1);
w2.Dispose();
oneThousand.Dispose();
DirectoryReader oneMillion = DirectoryReader.Open(dir2);
subReaders = new IndexReader[2000];
Arrays.Fill(subReaders, oneMillion);
mr = new MultiReader(subReaders);
BaseDirectoryWrapper dir3 = NewFSDirectory(CreateTempDir("2BPostingsBytes3"));
if (dir3 is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir3).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
IndexWriter w3 = new IndexWriter(dir3, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
w3.AddIndexes(mr);
w3.ForceMerge(1);
w3.Dispose();
oneMillion.Dispose();
dir.Dispose();
dir2.Dispose();
dir3.Dispose();
}
public virtual void TestNorms_Renamed()
{
// tmp dir
System.String tempDir = System.IO.Path.GetTempPath();
if (tempDir == null)
{
throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test");
}
// test with a single index: index1
System.IO.FileInfo indexDir1 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex1"));
Directory dir1 = FSDirectory.Open(indexDir1);
norms = new System.Collections.ArrayList();
modifiedNorms = new System.Collections.ArrayList();
CreateIndex(dir1);
DoTestNorms(dir1);
// test with a single index: index2
System.Collections.ArrayList norms1 = norms;
System.Collections.ArrayList modifiedNorms1 = modifiedNorms;
int numDocNorms1 = numDocNorms;
norms = new System.Collections.ArrayList();
modifiedNorms = new System.Collections.ArrayList();
numDocNorms = 0;
System.IO.FileInfo indexDir2 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex2"));
Directory dir2 = FSDirectory.Open(indexDir2);
CreateIndex(dir2);
DoTestNorms(dir2);
// add index1 and index2 to a third index: index3
System.IO.FileInfo indexDir3 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex3"));
Directory dir3 = FSDirectory.Open(indexDir3);
CreateIndex(dir3);
IndexWriter iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.AddIndexes(new Directory[]{dir1, dir2});
iw.Close();
norms1.AddRange(norms);
norms = norms1;
modifiedNorms1.AddRange(modifiedNorms);
modifiedNorms = modifiedNorms1;
numDocNorms += numDocNorms1;
// test with index3
VerifyIndex(dir3);
DoTestNorms(dir3);
// now with optimize
iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.Optimize();
iw.Close();
VerifyIndex(dir3);
dir1.Close();
dir2.Close();
dir3.Close();
}
public virtual void TestExistingDeletes()
{
Directory[] dirs = new Directory[2];
for (int i = 0; i < dirs.Length; i++)
{
dirs[i] = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter writer = new IndexWriter(dirs[i], conf);
Document doc = new Document();
doc.Add(new StringField("id", "myid", Field.Store.NO));
writer.AddDocument(doc);
writer.Dispose();
}
IndexWriterConfig conf_ = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter writer_ = new IndexWriter(dirs[0], conf_);
// Now delete the document
writer_.DeleteDocuments(new Term("id", "myid"));
IndexReader r = DirectoryReader.Open(dirs[1]);
try
{
writer_.AddIndexes(r);
}
finally
{
r.Dispose();
}
writer_.Commit();
Assert.AreEqual(1, writer_.NumDocs(), "Documents from the incoming index should not have been deleted");
writer_.Dispose();
foreach (Directory dir in dirs)
{
dir.Dispose();
}
}
public virtual void TestAddOldIndexes()
{
foreach (string name in OldNames)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: old index " + name);
}
Directory targetDir = NewDirectory();
IndexWriter w = new IndexWriter(targetDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
w.AddIndexes(OldIndexDirs[name]);
if (VERBOSE)
{
Console.WriteLine("\nTEST: done adding indices; now close");
}
w.Dispose();
targetDir.Dispose();
}
}
public void AddIndexWriters(IndexWriter writer, Lucene.Net.Store.Directory ramdir)
{
if (writer!=null || ramdir!=null)
{
writer.AddIndexes(new Lucene.Net.Store.Directory[] { ramdir });
}
}
public virtual void TestMoreMerges()
{
// main directory
Directory dir = NewDirectory();
// auxiliary directory
Directory aux = NewDirectory();
Directory aux2 = NewDirectory();
SetUpDirs(dir, aux, true);
IndexWriter writer = NewWriter(aux2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.CREATE).SetMaxBufferedDocs(100).SetMergePolicy(NewLogMergePolicy(10)));
writer.AddIndexes(aux);
Assert.AreEqual(30, writer.MaxDoc);
Assert.AreEqual(3, writer.SegmentCount);
writer.Dispose();
IndexWriterConfig dontMergeConfig = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
writer = new IndexWriter(aux, dontMergeConfig);
for (int i = 0; i < 27; i++)
{
writer.DeleteDocuments(new Term("id", "" + i));
}
writer.Dispose();
IndexReader reader = DirectoryReader.Open(aux);
Assert.AreEqual(3, reader.NumDocs);
reader.Dispose();
dontMergeConfig = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
writer = new IndexWriter(aux2, dontMergeConfig);
for (int i = 0; i < 8; i++)
{
writer.DeleteDocuments(new Term("id", "" + i));
}
writer.Dispose();
reader = DirectoryReader.Open(aux2);
Assert.AreEqual(22, reader.NumDocs);
reader.Dispose();
writer = NewWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.APPEND).SetMaxBufferedDocs(6).SetMergePolicy(NewLogMergePolicy(4)));
writer.AddIndexes(aux, aux2);
Assert.AreEqual(1040, writer.MaxDoc);
Assert.AreEqual(1000, writer.GetDocCount(0));
writer.Dispose();
dir.Dispose();
aux.Dispose();
aux2.Dispose();
}
public virtual void TestEmptyIndexWithVectors()
{
Directory rd1 = NewDirectory();
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: make 1st writer");
}
IndexWriter iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document doc = new Document();
Field idField = NewTextField("id", "", Field.Store.NO);
doc.Add(idField);
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.StoreTermVectors = true;
doc.Add(NewField("test", "", customType));
idField.StringValue = "1";
iw.AddDocument(doc);
doc.Add(NewTextField("test", "", Field.Store.NO));
idField.StringValue = "2";
iw.AddDocument(doc);
iw.Dispose();
IndexWriterConfig dontMergeConfig = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
if (VERBOSE)
{
Console.WriteLine("\nTEST: make 2nd writer");
}
IndexWriter writer = new IndexWriter(rd1, dontMergeConfig);
writer.DeleteDocuments(new Term("id", "1"));
writer.Dispose();
IndexReader ir = DirectoryReader.Open(rd1);
Assert.AreEqual(2, ir.MaxDoc);
Assert.AreEqual(1, ir.NumDocs);
ir.Dispose();
iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.APPEND));
iw.ForceMerge(1);
iw.Dispose();
}
Directory rd2 = NewDirectory();
{
IndexWriter iw = new IndexWriter(rd2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document doc = new Document();
iw.AddDocument(doc);
iw.Dispose();
}
Directory rdOut = NewDirectory();
IndexWriter iwOut = new IndexWriter(rdOut, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
DirectoryReader reader1, reader2;
ParallelAtomicReader pr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(reader1 = DirectoryReader.Open(rd1)), SlowCompositeReaderWrapper.Wrap(reader2 = DirectoryReader.Open(rd2)));
// When unpatched, Lucene crashes here with an ArrayIndexOutOfBoundsException (caused by TermVectorsWriter)
iwOut.AddIndexes(pr);
// ParallelReader closes any IndexReader you added to it:
pr.Dispose();
// assert subreaders were closed
Assert.AreEqual(0, reader1.RefCount);
Assert.AreEqual(0, reader2.RefCount);
rd1.Dispose();
rd2.Dispose();
iwOut.ForceMerge(1);
iwOut.Dispose();
rdOut.Dispose();
}
public virtual void TestEmptyIndexWithVectors()
{
Directory rd1 = NewDirectory();
{
if (Verbose)
{
Console.WriteLine("\nTEST: make 1st writer");
}
IndexWriter iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
Document doc = new Document();
Field idField = NewTextField("id", "", Field.Store.NO);
doc.Add(idField);
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.StoreTermVectors = true;
doc.Add(NewField("test", "", customType));
idField.SetStringValue("1");
iw.AddDocument(doc);
doc.Add(NewTextField("test", "", Field.Store.NO));
idField.SetStringValue("2");
iw.AddDocument(doc);
iw.Dispose();
IndexWriterConfig dontMergeConfig = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))).SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
if (Verbose)
{
Console.WriteLine("\nTEST: make 2nd writer");
}
IndexWriter writer = new IndexWriter(rd1, dontMergeConfig);
writer.DeleteDocuments(new Term("id", "1"));
writer.Dispose();
IndexReader ir = DirectoryReader.Open(rd1);
Assert.AreEqual(2, ir.MaxDoc);
Assert.AreEqual(1, ir.NumDocs);
ir.Dispose();
iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND));
iw.ForceMerge(1);
iw.Dispose();
}
Directory rd2 = NewDirectory();
{
IndexWriter iw = new IndexWriter(rd2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
Document doc = new Document();
iw.AddDocument(doc);
iw.Dispose();
}
Directory rdOut = NewDirectory();
IndexWriter iwOut = new IndexWriter(rdOut, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
DirectoryReader reader1, reader2;
ParallelAtomicReader pr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(reader1 = DirectoryReader.Open(rd1)), SlowCompositeReaderWrapper.Wrap(reader2 = DirectoryReader.Open(rd2)));
// When unpatched, Lucene crashes here with an ArrayIndexOutOfBoundsException (caused by TermVectorsWriter)
iwOut.AddIndexes(pr);
// ParallelReader closes any IndexReader you added to it:
pr.Dispose();
// assert subreaders were closed
Assert.AreEqual(0, reader1.RefCount);
Assert.AreEqual(0, reader2.RefCount);
rd1.Dispose();
rd2.Dispose();
iwOut.ForceMerge(1);
iwOut.Dispose();
rdOut.Dispose();
}
public virtual void TestHangOnClose()
{
Directory dir = NewDirectory();
LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();
lmp.NoCFSRatio = 0.0;
lmp.MergeFactor = 100;
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(5).SetMergePolicy(lmp));
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
doc.Add(NewField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType));
for (int i = 0; i < 60; i++)
{
writer.AddDocument(doc);
}
Document doc2 = new Document();
FieldType customType2 = new FieldType();
customType2.Stored = true;
doc2.Add(NewField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType2));
doc2.Add(NewField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType2));
doc2.Add(NewField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType2));
doc2.Add(NewField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType2));
for (int i = 0; i < 10; i++)
{
writer.AddDocument(doc2);
}
writer.Dispose();
Directory dir2 = NewDirectory();
lmp = new LogByteSizeMergePolicy();
lmp.MinMergeMB = 0.0001;
lmp.NoCFSRatio = 0.0;
lmp.MergeFactor = 4;
writer = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(new SerialMergeScheduler()).SetMergePolicy(lmp));
writer.AddIndexes(dir);
writer.Dispose();
dir.Dispose();
dir2.Dispose();
}
