| public SetMaxBufferedDocs ( int maxBufferedDocs ) : void | ||
| maxBufferedDocs | int | |
| return | void | |
public void TestRollbackIntegrityWithBufferFlush()
{
Directory dir = new MockRAMDirectory();
IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
for (int i = 0; i < 5; i++)
{
Document doc = new Document();
doc.Add(new Field("pk", i.ToString(), Field.Store.YES, Field.Index.ANALYZED_NO_NORMS));
w.AddDocument(doc);
}
w.Close();
// If buffer size is small enough to cause a flush, errors ensue...
w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
w.SetMaxBufferedDocs(2);
Term pkTerm = new Term("pk", "");
for (int i = 0; i < 3; i++)
{
Document doc = new Document();
String value = i.ToString();
doc.Add(new Field("pk", value, Field.Store.YES, Field.Index.ANALYZED_NO_NORMS));
doc.Add(new Field("text", "foo", Field.Store.YES, Field.Index.ANALYZED_NO_NORMS));
w.UpdateDocument(pkTerm.CreateTerm(value), doc);
}
w.Rollback();
IndexReader r = IndexReader.Open(dir, true);
Assert.AreEqual(5, r.NumDocs(), "index should contain same number of docs post rollback");
r.Close();
dir.Close();
}
/// <summary>
/// Open the index in the given directory and create a new index of word frequency for the
/// given index.</summary>
public void BuildAutoCompleteIndex()
{
// use a custom analyzer so we can do EdgeNGramFiltering
var analyzer = new AutoCompleteAnalyzer();
using (var writer = new IndexWriter(m_directory, analyzer, true, IndexWriter.MaxFieldLength.LIMITED))
{
writer.MergeFactor = 300;
writer.SetMaxBufferedDocs(150);
// go through every word, storing the original word (incl. n-grams)
// and the number of times it occurs
foreach (var hotel in _hotels)
{
if (hotel.Name.Length < 3)
continue; // too short we bail but "too long" is fine...
// ok index the word
// use the number of documents this word appears in
int freq = hotel.SearchCount;
var doc = MakeDocument(hotel, freq);
writer.AddDocument(doc);
}
writer.Optimize();
}
// re-open our reader
ReplaceSearcher();
}
public virtual void TestSorting()
{
Directory directory = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(2);
writer.SetMergeFactor(1000);
writer.AddDocument(Adoc(new System.String[]{"id", "a", "title", "ipod", "str_s", "a"}));
writer.AddDocument(Adoc(new System.String[]{"id", "b", "title", "ipod ipod", "str_s", "b"}));
writer.AddDocument(Adoc(new System.String[]{"id", "c", "title", "ipod ipod ipod", "str_s", "c"}));
writer.AddDocument(Adoc(new System.String[]{"id", "x", "title", "boosted", "str_s", "x"}));
writer.AddDocument(Adoc(new System.String[]{"id", "y", "title", "boosted boosted", "str_s", "y"}));
writer.AddDocument(Adoc(new System.String[]{"id", "z", "title", "boosted boosted boosted", "str_s", "z"}));
IndexReader r = writer.GetReader();
writer.Close();
IndexSearcher searcher = new IndexSearcher(r);
RunTest(searcher, true);
RunTest(searcher, false);
searcher.Close();
r.Close();
directory.Close();
}
public void ApplyToWriter(IndexWriter writer)
{
try
{
if (MergeFactor != null)
{
writer.SetMergeFactor((int) MergeFactor);
}
if (MaxMergeDocs != null)
{
writer.SetMaxMergeDocs((int) MaxMergeDocs);
}
if (MaxBufferedDocs != null)
{
writer.SetMaxBufferedDocs((int) MaxBufferedDocs);
}
if (RamBufferSizeMb != null)
{
writer.SetRAMBufferSizeMB((int) RamBufferSizeMb);
}
if (TermIndexInterval != null)
{
writer.SetTermIndexInterval((int) TermIndexInterval);
}
}
catch (ArgumentOutOfRangeException)
{
// TODO: Log it
}
}
internal virtual void BuildDir(Directory dir, int nDocs, int maxFields, int maxFieldLen)
{
IndexWriter iw = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);
iw.SetMaxBufferedDocs(10);
for (int j = 0; j < nDocs; j++)
{
Document d = new Document();
int nFields = r.Next(maxFields);
for (int i = 0; i < nFields; i++)
{
int flen = r.Next(maxFieldLen);
System.Text.StringBuilder sb = new System.Text.StringBuilder("^ ");
while (sb.Length < flen)
{
sb.Append(' ').Append(words[r.Next(words.Length)]);
}
sb.Append(" $");
Field.Store store = Field.Store.YES; // make random later
Field.Index index = Field.Index.ANALYZED; // make random later
d.Add(new Field("f" + i, sb.ToString(), store, index));
}
iw.AddDocument(d, null);
}
iw.Close();
}
public virtual void TestCommitUserData()
{
RAMDirectory d = new MockRAMDirectory();
System.Collections.Generic.IDictionary<string, string> commitUserData = new System.Collections.Generic.Dictionary<string,string>();
commitUserData["foo"] = "fighters";
// set up writer
IndexWriter writer = new IndexWriter(d, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(2);
for (int i = 0; i < 27; i++)
AddDocumentWithFields(writer);
writer.Close();
IndexReader r = IndexReader.Open(d, false);
r.DeleteDocument(5);
r.Flush(commitUserData);
r.Close();
SegmentInfos sis = new SegmentInfos();
sis.Read(d);
IndexReader r2 = IndexReader.Open(d, false);
IndexCommit c = r.IndexCommit;
Assert.AreEqual(c.UserData, commitUserData);
Assert.AreEqual(sis.GetCurrentSegmentFileName(), c.SegmentsFileName);
Assert.IsTrue(c.Equals(r.IndexCommit));
// Change the index
writer = new IndexWriter(d, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(2);
for (int i = 0; i < 7; i++)
AddDocumentWithFields(writer);
writer.Close();
IndexReader r3 = r2.Reopen();
Assert.IsFalse(c.Equals(r3.IndexCommit));
Assert.IsFalse(r2.IndexCommit.IsOptimized);
r3.Close();
writer = new IndexWriter(d, new StandardAnalyzer(Util.Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.LIMITED);
writer.Optimize();
writer.Close();
r3 = r2.Reopen();
Assert.IsTrue(r3.IndexCommit.IsOptimized);
r2.Close();
r3.Close();
d.Close();
}
public virtual void TestFlushExceptions()
{
MockRAMDirectory directory = new MockRAMDirectory();
FailOnlyOnFlush failure = new FailOnlyOnFlush();
directory.FailOn(failure);
IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED);
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
int extraCount = 0;
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 20; j++)
{
idField.SetValue(System.Convert.ToString(i*20 + j));
writer.AddDocument(doc);
}
while (true)
{
// must cycle here because sometimes the merge flushes
// the doc we just added and so there's nothing to
// flush, and we don't hit the exception
writer.AddDocument(doc);
failure.SetDoFail();
try
{
writer.Flush(true, false, true);
if (failure.hitExc)
Assert.Fail("failed to hit IOException");
extraCount++;
}
catch (System.IO.IOException ioe)
{
failure.ClearDoFail();
break;
}
}
}
writer.Close();
IndexReader reader = IndexReader.Open(directory, true);
Assert.AreEqual(200 + extraCount, reader.NumDocs());
reader.Close();
directory.Close();
}
/// <summary>
/// Получение класса для построения индекса, оптимизированного под обработку больших объёмов данных
/// </summary>
public static IndexWriter CreateIndexWriter()
{
var indexPath = GetIndexDir();
var w =
new IndexWriter(
indexPath,
new RussianAnalyzer(Version.LUCENE_30),
!IndexReader.IndexExists(indexPath),
IndexWriter.MaxFieldLength.UNLIMITED);
// optimizing
w.SetMaxBufferedDocs(4*1024);
//w.SetMergeFactor(30);
return w;
}
private IndexWriter InitIndex(MockRAMDirectory dir)
{
dir.SetLockFactory(NoLockFactory.Instance);
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
//writer.setMaxBufferedDocs(2);
writer.SetMaxBufferedDocs(10);
((ConcurrentMergeScheduler) writer.MergeScheduler).SetSuppressExceptions();
Document doc = new Document();
doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field("id", "0", Field.Store.YES, Field.Index.ANALYZED));
for (int i = 0; i < 157; i++)
writer.AddDocument(doc);
return writer;
}
public virtual void TestNormalCase()
{
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
writer.MergeFactor = 10;
writer.SetMergePolicy(new LogDocMergePolicy(writer));
for (int i = 0; i < 100; i++)
{
AddDoc(writer);
CheckInvariants(writer);
}
writer.Close();
}
private void CreateIndex(int numHits)
{
int numDocs = 500;
Directory directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true);
writer.SetMaxBufferedDocs(10);
for (int i = 0; i < numDocs; i++)
{
Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
System.String content;
if (i % (numDocs / numHits) == 0)
{
// add a document that matches the query "term1 term2"
content = this.term1 + " " + this.term2;
}
else if (i % 15 == 0)
{
// add a document that only contains term1
content = this.term1 + " " + this.term1;
}
else
{
// add a document that contains term2 but not term 1
content = this.term3 + " " + this.term2;
}
doc.Add(new Field(this.field, content, Field.Store.YES, Field.Index.TOKENIZED));
writer.AddDocument(doc);
}
// make sure the index has only a single segment
writer.Optimize();
writer.Close();
// the index is a single segment, thus IndexReader.open() returns an instance of SegmentReader
SegmentReader reader = (SegmentReader) IndexReader.Open(directory);
// we decorate the proxStream with a wrapper class that allows to count the number of calls of seek()
reader.ProxStream_ForNUnitTest = new SeeksCountingStream(this, reader.ProxStream_ForNUnitTest);
this.searcher = new IndexSearcher(reader);
}
public virtual void TestFlushExceptions()
{
MockRAMDirectory directory = new MockRAMDirectory();
FailOnlyOnFlush failure = new FailOnlyOnFlush();
directory.FailOn(failure);
IndexWriter writer = new IndexWriter(directory, true, ANALYZER, true);
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 20; j++)
{
idField.SetValue(System.Convert.ToString(i * 20 + j));
writer.AddDocument(doc);
}
writer.AddDocument(doc);
failure.SetDoFail();
try
{
writer.Flush();
Assert.Fail("failed to hit IOException");
}
catch (System.IO.IOException ioe)
{
failure.ClearDoFail();
}
}
writer.Close();
IndexReader reader = IndexReader.Open(directory);
Assert.AreEqual(200, reader.NumDocs());
reader.Close();
directory.Close();
}
public virtual void TestDeletedDocs()
{
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 19; i++)
{
writer.AddDocument(doc);
}
writer.Close();
IndexReader reader = IndexReader.Open(dir);
reader.DeleteDocument(5);
reader.Close();
CheckIndex.out_Renamed = new System.IO.StringWriter();
bool condition = CheckIndex.Check(dir, false);
String message = CheckIndex.out_Renamed.ToString();
Assert.IsTrue(condition, message);
}
public virtual void TestIndexing()
{
Directory mainDir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(mainDir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
writer.UseCompoundFile = false;
IndexReader reader = writer.GetReader(); // start pooling readers
reader.Close();
writer.MergeFactor = 2;
writer.SetMaxBufferedDocs(10);
RunThread[] indexThreads = new RunThread[4];
for (int x = 0; x < indexThreads.Length; x++)
{
indexThreads[x] = new RunThread(this, x % 2, writer);
indexThreads[x].Name = "Thread " + x;
indexThreads[x].Start();
}
long startTime = (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond);
long duration = 5 * 1000;
while (((DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) - startTime) < duration)
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 100));
}
int delCount = 0;
int addCount = 0;
for (int x = 0; x < indexThreads.Length; x++)
{
indexThreads[x].run_Renamed_Field = false;
Assert.IsTrue(indexThreads[x].ex == null);
addCount += indexThreads[x].addCount;
delCount += indexThreads[x].delCount;
}
for (int x = 0; x < indexThreads.Length; x++)
{
indexThreads[x].Join();
}
//System.out.println("addCount:"+addCount);
//System.out.println("delCount:"+delCount);
writer.Close();
mainDir.Close();
}
public virtual void TestNoOverMerge()
{
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
writer.MergeFactor = 10;
writer.SetMergePolicy(new LogDocMergePolicy(writer));
bool noOverMerge = false;
for (int i = 0; i < 100; i++)
{
AddDoc(writer);
CheckInvariants(writer);
if (writer.GetNumBufferedDocuments() + writer.GetSegmentCount() >= 18)
{
noOverMerge = true;
}
}
Assert.IsTrue(noOverMerge);
writer.Close();
}
public virtual void TestBothDeletes()
{
Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
modifier.SetMaxBufferedDocs(100);
modifier.SetMaxBufferedDeleteTerms(100);
int id = 0;
int value_Renamed = 100;
for (int i = 0; i < 5; i++)
{
AddDoc(modifier, ++id, value_Renamed);
}
value_Renamed = 200;
for (int i = 0; i < 5; i++)
{
AddDoc(modifier, ++id, value_Renamed);
}
modifier.Commit();
for (int i = 0; i < 5; i++)
{
AddDoc(modifier, ++id, value_Renamed);
}
modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
modifier.Commit();
IndexReader reader = IndexReader.Open(dir, true);
Assert.AreEqual(5, reader.NumDocs());
modifier.Close();
}
public virtual void TestNoTailSegments()
{
// main directory
Directory dir = new RAMDirectory();
// auxiliary directory
Directory aux = new RAMDirectory();
SetUpDirs(dir, aux);
IndexWriter writer = NewWriter(dir, false);
writer.SetMaxBufferedDocs(10);
writer.MergeFactor = 4;
AddDocs(writer, 10);
writer.AddIndexesNoOptimize(null, new Directory[] { aux });
Assert.AreEqual(1040, writer.MaxDoc());
Assert.AreEqual(2, writer.GetSegmentCount());
Assert.AreEqual(1000, writer.GetDocCount(0));
writer.Close();
// make sure the index is correct
VerifyNumDocs(dir, 1040);
}
private void CreateIndex(Directory dir)
{
IndexWriter iw = new IndexWriter(dir, anlzr, true, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.SetSimilarity(similarityOne);
iw.SetUseCompoundFile(true);
iw.Close();
}
public virtual void TestKeepLastNDeletionPolicyWithCreates()
{
int N = 10;
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
writer.Close();
Term searchTerm = new Term("content", "aaa");
Query query = new TermQuery(searchTerm);
for (int i = 0; i < N + 1; i++)
{
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
for (int j = 0; j < 17; j++)
{
AddDoc(writer);
}
// this is a commit when autoCommit=false:
writer.Close();
IndexReader reader = IndexReader.Open(dir, policy);
reader.DeleteDocument(3);
reader.SetNorm(5, "content", 2.0F);
IndexSearcher searcher = new IndexSearcher(reader);
ScoreDoc[] hits = searcher.Search(query, null, 1000).scoreDocs;
Assert.AreEqual(16, hits.Length);
// this is a commit when autoCommit=false:
reader.Close();
searcher.Close();
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
// This will not commit: there are no changes
// pending because we opened for "create":
writer.Close();
}
Assert.AreEqual(1 + 3 * (N + 1), policy.numOnInit);
if (!autoCommit)
{
Assert.AreEqual(3 * (N + 1), policy.numOnCommit);
}
IndexSearcher searcher2 = new IndexSearcher(dir);
ScoreDoc[] hits2 = searcher2.Search(query, null, 1000).scoreDocs;
Assert.AreEqual(0, hits2.Length);
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
int expectedCount = 0;
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = IndexReader.Open(dir);
// Work backwards in commits on what the expected
// count should be. Only check this in the
// autoCommit false case:
if (!autoCommit)
{
searcher2 = new IndexSearcher(reader);
hits2 = searcher2.Search(query, null, 1000).scoreDocs;
Assert.AreEqual(expectedCount, hits2.Length);
searcher2.Close();
if (expectedCount == 0)
{
expectedCount = 16;
}
else if (expectedCount == 16)
{
expectedCount = 17;
}
else if (expectedCount == 17)
{
expectedCount = 0;
}
}
reader.Close();
if (i == N)
{
Assert.Fail("should have failed on commits before last " + N);
}
}
catch (System.IO.IOException e)
{
if (i != N)
{
throw e;
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Close();
}
}
public virtual void TestDeleteLeftoverFiles()
{
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
int i;
for (i = 0; i < 35; i++)
{
AddDoc(writer, i);
}
writer.SetUseCompoundFile(false);
for (; i < 45; i++)
{
AddDoc(writer, i);
}
writer.Close();
// Delete one doc so we get a .del file:
IndexReader reader = IndexReader.Open(dir);
Term searchTerm = new Term("id", "7");
int delCount = reader.DeleteDocuments(searchTerm);
Assert.AreEqual(1, delCount, "didn't delete the right number of documents");
// Set one norm so we get a .s0 file:
reader.SetNorm(21, "content", (float)1.5);
reader.Close();
// Now, artificially create an extra .del file & extra
// .s0 file:
System.String[] files = dir.ListAll();
/*
* for(int j=0;j<files.length;j++) {
* System.out.println(j + ": " + files[j]);
* }
*/
// The numbering of fields can vary depending on which
// JRE is in use. On some JREs we see content bound to
// field 0; on others, field 1. So, here we have to
// figure out which field number corresponds to
// "content", and then set our expected file names below
// accordingly:
CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs");
FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
int contentFieldIndex = -1;
for (i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.name_ForNUnit.Equals("content"))
{
contentFieldIndex = i;
break;
}
}
cfsReader.Close();
Assert.IsTrue(contentFieldIndex != -1, "could not locate the 'content' field number in the _2.cfs segment");
System.String normSuffix = "s" + contentFieldIndex;
// Create a bogus separate norms file for a
// segment/field that actually has a separate norms file
// already:
CopyFile(dir, "_2_1." + normSuffix, "_2_2." + normSuffix);
// Create a bogus separate norms file for a
// segment/field that actually has a separate norms file
// already, using the "not compound file" extension:
CopyFile(dir, "_2_1." + normSuffix, "_2_2.f" + contentFieldIndex);
// Create a bogus separate norms file for a
// segment/field that does not have a separate norms
// file already:
CopyFile(dir, "_2_1." + normSuffix, "_1_1." + normSuffix);
// Create a bogus separate norms file for a
// segment/field that does not have a separate norms
// file already using the "not compound file" extension:
CopyFile(dir, "_2_1." + normSuffix, "_1_1.f" + contentFieldIndex);
// Create a bogus separate del file for a
// segment that already has a separate del file:
CopyFile(dir, "_0_1.del", "_0_2.del");
// Create a bogus separate del file for a
// segment that does not yet have a separate del file:
CopyFile(dir, "_0_1.del", "_1_1.del");
// Create a bogus separate del file for a
// non-existent segment:
CopyFile(dir, "_0_1.del", "_188_1.del");
// Create a bogus segment file:
CopyFile(dir, "_0.cfs", "_188.cfs");
// Create a bogus fnm file when the CFS already exists:
CopyFile(dir, "_0.cfs", "_0.fnm");
// Create a deletable file:
CopyFile(dir, "_0.cfs", "deletable");
// Create some old segments file:
CopyFile(dir, "segments_3", "segments");
CopyFile(dir, "segments_3", "segments_2");
// Create a bogus cfs file shadowing a non-cfs segment:
CopyFile(dir, "_2.cfs", "_3.cfs");
System.String[] filesPre = dir.ListAll();
// Open & close a writer: it should delete the above 4
// files and nothing more:
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
writer.Close();
System.String[] files2 = dir.ListAll();
dir.Close();
System.Array.Sort(files);
System.Array.Sort(files2);
System.Collections.Hashtable dif = DifFiles(files, files2);
if (!SupportClass.CollectionsHelper.Equals(files, files2))
{
Assert.Fail("IndexFileDeleter failed to delete unreferenced extra files: should have deleted " + (filesPre.Length - files.Length) + " files but only deleted " + (filesPre.Length - files2.Length) + "; expected files:\n " + AsString(files) + "\n actual files:\n " + AsString(files2) + "\ndif: " + SupportClass.CollectionsHelper.CollectionToString(dif));
}
}
public virtual void TestNorms()
{
// tmp dir
System.String tempDir = System.IO.Path.GetTempPath();
if (tempDir == null)
{
throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test");
}
// test with a single index: index1
System.IO.FileInfo indexDir1 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex1"));
Directory dir1 = FSDirectory.Open(indexDir1);
IndexWriter.Unlock(dir1);
norms = new System.Collections.ArrayList();
modifiedNorms = new System.Collections.ArrayList();
CreateIndex(dir1);
DoTestNorms(dir1);
// test with a single index: index2
System.Collections.ArrayList norms1 = norms;
System.Collections.ArrayList modifiedNorms1 = modifiedNorms;
int numDocNorms1 = numDocNorms;
norms = new System.Collections.ArrayList();
modifiedNorms = new System.Collections.ArrayList();
numDocNorms = 0;
System.IO.FileInfo indexDir2 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex2"));
Directory dir2 = FSDirectory.Open(indexDir2);
CreateIndex(dir2);
DoTestNorms(dir2);
// add index1 and index2 to a third index: index3
System.IO.FileInfo indexDir3 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex3"));
Directory dir3 = FSDirectory.Open(indexDir3);
CreateIndex(dir3);
IndexWriter iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.AddIndexes(new Directory[] { dir1, dir2 });
iw.Close();
norms1.AddRange(norms);
norms = norms1;
modifiedNorms1.AddRange(modifiedNorms);
modifiedNorms = modifiedNorms1;
numDocNorms += numDocNorms1;
// test with index3
VerifyIndex(dir3);
DoTestNorms(dir3);
// now with optimize
iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.Optimize();
iw.Close();
VerifyIndex(dir3);
dir1.Close();
dir2.Close();
dir3.Close();
}
public override void DoWork()
{
IndexWriter writer1 = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
writer1.SetMaxBufferedDocs(3);
writer1.MergeFactor = 2;
((ConcurrentMergeScheduler) writer1.MergeScheduler).SetSuppressExceptions();
IndexWriter writer2 = new IndexWriter(dir2, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
// Intentionally use different params so flush/merge
// happen @ different times
writer2.SetMaxBufferedDocs(2);
writer2.MergeFactor = 3;
((ConcurrentMergeScheduler) writer2.MergeScheduler).SetSuppressExceptions();
Update(writer1);
Update(writer2);
TestTransactions.doFail = true;
try
{
lock (lock_Renamed)
{
try
{
writer1.PrepareCommit();
}
catch (System.Exception t)
{
writer1.Rollback();
writer2.Rollback();
return ;
}
try
{
writer2.PrepareCommit();
}
catch (System.Exception t)
{
writer1.Rollback();
writer2.Rollback();
return ;
}
writer1.Commit();
writer2.Commit();
}
}
finally
{
TestTransactions.doFail = false;
}
writer1.Close();
writer2.Close();
}
internal virtual void AddDocs(Directory dir, int ndocs, System.String field, System.String val, int maxTF, float percentDocs)
{
System.Random random = NewRandom();
RepeatingTokenStream ts = new RepeatingTokenStream(val);
Analyzer analyzer = new AnonymousClassAnalyzer(random, percentDocs, ts, maxTF, this);
Document doc = new Document();
doc.Add(new Field(field, val, Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(100);
writer.SetMergeFactor(100);
for (int i = 0; i < ndocs; i++)
{
writer.AddDocument(doc);
}
writer.Optimize();
writer.Close();
}
public virtual void runTest(Directory directory, MergeScheduler merger)
{
IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED, null);
writer.SetMaxBufferedDocs(2);
if (merger != null)
{
writer.SetMergeScheduler(merger, null);
}
for (int iter = 0; iter < NUM_ITER; iter++)
{
int iterFinal = iter;
writer.MergeFactor = 1000;
for (int i = 0; i < 200; i++)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
d.Add(new Field("contents", English.IntToEnglish(i), Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(d, null);
}
writer.MergeFactor = 4;
//writer.setInfoStream(System.out);
ThreadClass[] threads = new ThreadClass[NUM_THREADS];
for (int i = 0; i < NUM_THREADS; i++)
{
int iFinal = i;
IndexWriter writerFinal = writer;
threads[i] = new AnonymousClassThread(writerFinal, iFinal, iterFinal, this);
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Start();
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Join();
}
Assert.IsTrue(!failed);
int expectedDocCount = (int)((1 + iter) * (200 + 8 * NUM_ITER2 * (NUM_THREADS / 2.0) * (1 + NUM_THREADS)));
// System.out.println("TEST: now index=" + writer.segString());
Assert.AreEqual(expectedDocCount, writer.MaxDoc());
writer.Close();
writer = new IndexWriter(directory, ANALYZER, false, IndexWriter.MaxFieldLength.UNLIMITED, null);
writer.SetMaxBufferedDocs(2);
IndexReader reader = IndexReader.Open(directory, true, null);
Assert.IsTrue(reader.IsOptimized());
Assert.AreEqual(expectedDocCount, reader.NumDocs());
reader.Close();
}
writer.Close();
}
public virtual void TestKeepNoneOnInitDeletionPolicy()
{
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy(this);
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
for (int i = 0; i < 107; i++)
{
AddDoc(writer);
}
writer.Close();
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
writer.SetUseCompoundFile(useCompoundFile);
writer.Optimize();
writer.Close();
Assert.AreEqual(2, policy.numOnInit);
if (autoCommit)
{
Assert.IsTrue(policy.numOnCommit > 2);
}
else
{
// If we are not auto committing then there should
// be exactly 2 commits (one per close above):
Assert.AreEqual(2, policy.numOnCommit);
}
// Simplistic check: just verify the index is in fact
// readable:
IndexReader reader = IndexReader.Open(dir);
reader.Close();
dir.Close();
}
}
public virtual void TestKeepAllDeletionPolicy()
{
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(this);
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
for (int i = 0; i < 107; i++)
{
AddDoc(writer);
}
writer.Close();
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
writer.SetUseCompoundFile(useCompoundFile);
writer.Optimize();
writer.Close();
Assert.AreEqual(2, policy.numOnInit);
if (autoCommit)
{
Assert.IsTrue(policy.numOnCommit > 2);
}
else
{
// If we are not auto committing then there should
// be exactly 2 commits (one per close above):
Assert.AreEqual(2, policy.numOnCommit);
}
// Simplistic check: just verify all segments_N's still
// exist, and, I can open a reader on each:
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
while (gen > 0)
{
IndexReader reader = IndexReader.Open(dir);
reader.Close();
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
if (gen > 0)
{
// Now that we've removed a commit point, which
// should have orphan'd at least one index file.
// Open & close a writer and assert that it
// actually removed something:
int preCount = dir.List().Length;
writer = new IndexWriter(dir, false, new WhitespaceAnalyzer(), false, policy);
writer.Close();
int postCount = dir.List().Length;
Assert.IsTrue(postCount < preCount);
}
}
dir.Close();
}
}
public virtual void TestDeletedDocs()
{
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 19; i++)
{
writer.AddDocument(doc);
}
writer.Close();
IndexReader reader = IndexReader.Open(dir, false);
reader.DeleteDocument(5);
reader.Close();
System.IO.MemoryStream bos = new System.IO.MemoryStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.SetInfoStream(new System.IO.StreamWriter(bos));
//checker.setInfoStream(System.out);
CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method();
if (indexStatus.clean == false)
{
System.Console.Out.WriteLine("CheckIndex failed");
char[] tmpChar;
byte[] tmpByte;
tmpByte = bos.GetBuffer();
tmpChar = new char[bos.Length];
System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length);
System.Console.Out.WriteLine(new System.String(tmpChar));
Assert.Fail();
}
CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus) indexStatus.segmentInfos[0];
Assert.IsTrue(seg.openReaderPassed);
Assert.IsNotNull(seg.diagnostics);
Assert.IsNotNull(seg.fieldNormStatus);
Assert.IsNull(seg.fieldNormStatus.error);
Assert.AreEqual(1, seg.fieldNormStatus.totFields);
Assert.IsNotNull(seg.termIndexStatus);
Assert.IsNull(seg.termIndexStatus.error);
Assert.AreEqual(1, seg.termIndexStatus.termCount);
Assert.AreEqual(19, seg.termIndexStatus.totFreq);
Assert.AreEqual(18, seg.termIndexStatus.totPos);
Assert.IsNotNull(seg.storedFieldStatus);
Assert.IsNull(seg.storedFieldStatus.error);
Assert.AreEqual(18, seg.storedFieldStatus.docCount);
Assert.AreEqual(18, seg.storedFieldStatus.totFields);
Assert.IsNotNull(seg.termVectorStatus);
Assert.IsNull(seg.termVectorStatus.error);
Assert.AreEqual(18, seg.termVectorStatus.docCount);
Assert.AreEqual(18, seg.termVectorStatus.totVectors);
Assert.IsTrue(seg.diagnostics.Count > 0);
List<string> onlySegments = new List<string>();
onlySegments.Add("_0");
Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true);
}
public virtual void TestDeleteAll()
{
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = (0 == pass);
Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
modifier.SetMaxBufferedDocs(2);
modifier.SetMaxBufferedDeleteTerms(2);
int id = 0;
int value_Renamed = 100;
for (int i = 0; i < 7; i++)
{
AddDoc(modifier, ++id, value_Renamed);
}
modifier.Commit();
IndexReader reader = IndexReader.Open(dir);
Assert.AreEqual(7, reader.NumDocs());
reader.Close();
// Add 1 doc (so we will have something buffered)
AddDoc(modifier, 99, value_Renamed);
// Delete all
modifier.DeleteAll();
// Delete all shouldn't be on disk yet
reader = IndexReader.Open(dir);
Assert.AreEqual(7, reader.NumDocs());
reader.Close();
// Add a doc and update a doc (after the deleteAll, before the commit)
AddDoc(modifier, 101, value_Renamed);
UpdateDoc(modifier, 102, value_Renamed);
// commit the delete all
modifier.Commit();
// Validate there are no docs left
reader = IndexReader.Open(dir);
Assert.AreEqual(2, reader.NumDocs());
reader.Close();
modifier.Close();
dir.Close();
}
}
public virtual void TestBatchDeletes()
{
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = (0 == pass);
Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
modifier.SetMaxBufferedDocs(2);
modifier.SetMaxBufferedDeleteTerms(2);
int id = 0;
int value_Renamed = 100;
for (int i = 0; i < 7; i++)
{
AddDoc(modifier, ++id, value_Renamed);
}
modifier.Commit();
IndexReader reader = IndexReader.Open(dir);
Assert.AreEqual(7, reader.NumDocs());
reader.Close();
id = 0;
modifier.DeleteDocuments(new Term("id", System.Convert.ToString(++id)));
modifier.DeleteDocuments(new Term("id", System.Convert.ToString(++id)));
modifier.Commit();
reader = IndexReader.Open(dir);
Assert.AreEqual(5, reader.NumDocs());
reader.Close();
Term[] terms = new Term[3];
for (int i = 0; i < terms.Length; i++)
{
terms[i] = new Term("id", System.Convert.ToString(++id));
}
modifier.DeleteDocuments(terms);
modifier.Commit();
reader = IndexReader.Open(dir);
Assert.AreEqual(2, reader.NumDocs());
reader.Close();
modifier.Close();
dir.Close();
}
}
public virtual void TestBasic()
{
Directory dir = new MockRAMDirectory();
Analyzer analyzer = new StandardAnalyzer();
IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMergeFactor(2);
writer.SetMaxBufferedDocs(2);
writer.SetSimilarity(new SimpleSimilarity());
System.Text.StringBuilder sb = new System.Text.StringBuilder(265);
System.String term = "term";
for (int i = 0; i < 30; i++)
{
Document d = new Document();
sb.Append(term).Append(" ");
System.String content = sb.ToString();
Field noTf = new Field("noTf", content + (i % 2 == 0?"":" notf"), Field.Store.NO, Field.Index.ANALYZED);
noTf.SetOmitTermFreqAndPositions(true);
d.Add(noTf);
Field tf = new Field("tf", content + (i % 2 == 0?" tf":""), Field.Store.NO, Field.Index.ANALYZED);
d.Add(tf);
writer.AddDocument(d);
//System.out.println(d);
}
writer.Optimize();
// flush
writer.Close();
_TestUtil.CheckIndex(dir);
/*
* Verify the index
*/
Searcher searcher = new IndexSearcher(dir);
searcher.SetSimilarity(new SimpleSimilarity());
Term a = new Term("noTf", term);
Term b = new Term("tf", term);
Term c = new Term("noTf", "notf");
Term d2 = new Term("tf", "tf");
TermQuery q1 = new TermQuery(a);
TermQuery q2 = new TermQuery(b);
TermQuery q3 = new TermQuery(c);
TermQuery q4 = new TermQuery(d2);
searcher.Search(q1, new AnonymousClassCountingHitCollector(this));
//System.out.println(CountingHitCollector.getCount());
searcher.Search(q2, new AnonymousClassCountingHitCollector1(this));
//System.out.println(CountingHitCollector.getCount());
searcher.Search(q3, new AnonymousClassCountingHitCollector2(this));
//System.out.println(CountingHitCollector.getCount());
searcher.Search(q4, new AnonymousClassCountingHitCollector3(this));
//System.out.println(CountingHitCollector.getCount());
BooleanQuery bq = new BooleanQuery();
bq.Add(q1, Occur.MUST);
bq.Add(q4, Occur.MUST);
searcher.Search(bq, new AnonymousClassCountingHitCollector4(this));
Assert.IsTrue(15 == CountingHitCollector.GetCount());
searcher.Close();
dir.Close();
}
private void AddDocs(Directory dir, int ndocs, bool compound)
{
IndexWriter iw = new IndexWriter(dir, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.SetSimilarity(similarityOne);
iw.SetUseCompoundFile(compound);
for (int i = 0; i < ndocs; i++)
{
iw.AddDocument(NewDoc());
}
iw.Close();
}
public virtual void TestKeepLastNDeletionPolicy()
{
int N = 5;
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
Directory dir = new RAMDirectory();
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
for (int j = 0; j < N + 1; j++)
{
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
for (int i = 0; i < 17; i++)
{
AddDoc(writer);
}
writer.Optimize();
writer.Close();
}
Assert.IsTrue(policy.numDelete > 0);
Assert.AreEqual(N + 1, policy.numOnInit);
if (autoCommit)
{
Assert.IsTrue(policy.numOnCommit > 1);
}
else
{
Assert.AreEqual(N + 1, policy.numOnCommit);
}
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = IndexReader.Open(dir);
reader.Close();
if (i == N)
{
Assert.Fail("should have failed on commits prior to last " + N);
}
}
catch (System.IO.IOException e)
{
if (i != N)
{
throw e;
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Close();
}
}
public virtual void TestNorms_Renamed()
{
// tmp dir
System.String tempDir = System.IO.Path.GetTempPath();
if (tempDir == null)
{
throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test");
}
// test with a single index: index1
System.IO.FileInfo indexDir1 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex1"));
Directory dir1 = FSDirectory.Open(indexDir1);
norms = new System.Collections.ArrayList();
modifiedNorms = new System.Collections.ArrayList();
CreateIndex(dir1);
DoTestNorms(dir1);
// test with a single index: index2
System.Collections.ArrayList norms1 = norms;
System.Collections.ArrayList modifiedNorms1 = modifiedNorms;
int numDocNorms1 = numDocNorms;
norms = new System.Collections.ArrayList();
modifiedNorms = new System.Collections.ArrayList();
numDocNorms = 0;
System.IO.FileInfo indexDir2 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex2"));
Directory dir2 = FSDirectory.Open(indexDir2);
CreateIndex(dir2);
DoTestNorms(dir2);
// add index1 and index2 to a third index: index3
System.IO.FileInfo indexDir3 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex3"));
Directory dir3 = FSDirectory.Open(indexDir3);
CreateIndex(dir3);
IndexWriter iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.AddIndexes(new Directory[]{dir1, dir2});
iw.Close();
norms1.AddRange(norms);
norms = norms1;
modifiedNorms1.AddRange(modifiedNorms);
modifiedNorms = modifiedNorms1;
numDocNorms += numDocNorms1;
// test with index3
VerifyIndex(dir3);
DoTestNorms(dir3);
// now with optimize
iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.Optimize();
iw.Close();
VerifyIndex(dir3);
dir1.Close();
dir2.Close();
dir3.Close();
}
public virtual void TestKeepLastNDeletionPolicyWithCreates()
{
int N = 10;
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
writer.Close();
Term searchTerm = new Term("content", "aaa");
Query query = new TermQuery(searchTerm);
for (int i = 0; i < N + 1; i++)
{
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
for (int j = 0; j < 17; j++)
{
AddDoc(writer);
}
// this is a commit when autoCommit=false:
writer.Close();
IndexReader reader = IndexReader.Open(dir, policy);
reader.DeleteDocument(3);
reader.SetNorm(5, "content", 2.0F);
IndexSearcher searcher = new IndexSearcher(reader);
Hits hits = searcher.Search(query);
Assert.AreEqual(16, hits.Length());
// this is a commit when autoCommit=false:
reader.Close();
searcher.Close();
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
// This will not commit: there are no changes
// pending because we opened for "create":
writer.Close();
}
Assert.AreEqual(1 + 3 * (N + 1), policy.numOnInit);
if (autoCommit)
{
Assert.IsTrue(policy.numOnCommit > 3 * (N + 1) - 1);
}
else
{
Assert.AreEqual(2 * (N + 1), policy.numOnCommit);
}
IndexSearcher searcher2 = new IndexSearcher(dir);
Hits hits2 = searcher2.Search(query);
Assert.AreEqual(0, hits2.Length());
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
int expectedCount = 0;
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = IndexReader.Open(dir);
// Work backwards in commits on what the expected
// count should be. Only check this in the
// autoCommit false case:
if (!autoCommit)
{
searcher2 = new IndexSearcher(reader);
hits2 = searcher2.Search(query);
Assert.AreEqual(expectedCount, hits2.Length());
searcher2.Close();
if (expectedCount == 0)
{
expectedCount = 16;
}
else if (expectedCount == 16)
{
expectedCount = 17;
}
else if (expectedCount == 17)
{
expectedCount = 0;
}
}
reader.Close();
if (i == N)
{
Assert.Fail("should have failed on commits before last " + N);
}
}
catch (System.IO.IOException e)
{
if (i != N)
{
throw e;
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Close();
}
}
private void CreateIndex(int numHits)
{
int numDocs = 500;
Directory directory = new SeekCountingDirectory(this);
IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.UseCompoundFile = false;
writer.SetMaxBufferedDocs(10);
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
System.String content;
if (i % (numDocs / numHits) == 0)
{
// add a document that matches the query "term1 term2"
content = this.term1 + " " + this.term2;
}
else if (i % 15 == 0)
{
// add a document that only contains term1
content = this.term1 + " " + this.term1;
}
else
{
// add a document that contains term2 but not term 1
content = this.term3 + " " + this.term2;
}
doc.Add(new Field(this.field, content, Field.Store.YES, Field.Index.ANALYZED));
writer.AddDocument(doc);
}
// make sure the index has only a single segment
writer.Optimize();
writer.Close();
SegmentReader reader = SegmentReader.GetOnlySegmentReader(directory);
this.searcher = new IndexSearcher(reader);
}
public virtual void TestKeepAllDeletionPolicy()
{
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
// Never deletes a commit
KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(this);
Directory dir = new RAMDirectory();
policy.dir = dir;
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
writer.SetMergeScheduler(new SerialMergeScheduler());
for (int i = 0; i < 107; i++)
{
AddDoc(writer);
if (autoCommit && i % 10 == 0)
{
writer.Commit();
}
}
writer.Close();
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
writer.SetUseCompoundFile(useCompoundFile);
writer.Optimize();
writer.Close();
Assert.AreEqual(2, policy.numOnInit);
if (!autoCommit)
{
// If we are not auto committing then there should
// be exactly 2 commits (one per close above):
Assert.AreEqual(2, policy.numOnCommit);
}
// Test listCommits
System.Collections.ICollection commits = IndexReader.ListCommits(dir);
if (!autoCommit)
{
// 1 from opening writer + 2 from closing writer
Assert.AreEqual(3, commits.Count);
}
// 1 from opening writer + 2 from closing writer +
// 11 from calling writer.commit() explicitly above
else
{
Assert.AreEqual(14, commits.Count);
}
System.Collections.IEnumerator it = commits.GetEnumerator();
// Make sure we can open a reader on each commit:
while (it.MoveNext())
{
IndexCommit commit = (IndexCommit)it.Current;
IndexReader r = IndexReader.Open(commit, null);
r.Close();
}
// Simplistic check: just verify all segments_N's still
// exist, and, I can open a reader on each:
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
while (gen > 0)
{
IndexReader reader = IndexReader.Open(dir);
reader.Close();
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
if (gen > 0)
{
// Now that we've removed a commit point, which
// should have orphan'd at least one index file.
// Open & close a writer and assert that it
// actually removed something:
int preCount = dir.ListAll().Length;
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.LIMITED);
writer.Close();
int postCount = dir.ListAll().Length;
Assert.IsTrue(postCount < preCount);
}
}
dir.Close();
}
}
public virtual void TestQuery()
{
RAMDirectory dir = new RAMDirectory();
IndexWriter iw = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED, null);
iw.SetMaxBufferedDocs(2); // force multi-segment
AddDoc("one", iw, 1f);
AddDoc("two", iw, 20f);
AddDoc("three four", iw, 300f);
iw.Close();
IndexReader ir = IndexReader.Open((Directory)dir, false, null);
IndexSearcher is_Renamed = new IndexSearcher(ir);
ScoreDoc[] hits;
// assert with norms scoring turned off
hits = is_Renamed.Search(new MatchAllDocsQuery(), null, 1000, null).ScoreDocs;
Assert.AreEqual(3, hits.Length);
Assert.AreEqual(ir.Document(hits[0].Doc, null).Get("key", null), "one");
Assert.AreEqual(ir.Document(hits[1].Doc, null).Get("key", null), "two");
Assert.AreEqual(ir.Document(hits[2].Doc, null).Get("key", null), "three four");
// assert with norms scoring turned on
MatchAllDocsQuery normsQuery = new MatchAllDocsQuery("key");
hits = is_Renamed.Search(normsQuery, null, 1000, null).ScoreDocs;
Assert.AreEqual(3, hits.Length);
Assert.AreEqual(ir.Document(hits[0].Doc, null).Get("key", null), "three four");
Assert.AreEqual(ir.Document(hits[1].Doc, null).Get("key", null), "two");
Assert.AreEqual(ir.Document(hits[2].Doc, null).Get("key", null), "one");
// change norm & retest
ir.SetNorm(0, "key", 400f, null);
normsQuery = new MatchAllDocsQuery("key");
hits = is_Renamed.Search(normsQuery, null, 1000, null).ScoreDocs;
Assert.AreEqual(3, hits.Length);
Assert.AreEqual(ir.Document(hits[0].Doc, null).Get("key", null), "one");
Assert.AreEqual(ir.Document(hits[1].Doc, null).Get("key", null), "three four");
Assert.AreEqual(ir.Document(hits[2].Doc, null).Get("key", null), "two");
// some artificial queries to trigger the use of skipTo():
BooleanQuery bq = new BooleanQuery();
bq.Add(new MatchAllDocsQuery(), Occur.MUST);
bq.Add(new MatchAllDocsQuery(), Occur.MUST);
hits = is_Renamed.Search(bq, null, 1000, null).ScoreDocs;
Assert.AreEqual(3, hits.Length);
bq = new BooleanQuery();
bq.Add(new MatchAllDocsQuery(), Occur.MUST);
bq.Add(new TermQuery(new Term("key", "three")), Occur.MUST);
hits = is_Renamed.Search(bq, null, 1000, null).ScoreDocs;
Assert.AreEqual(1, hits.Length);
// delete a document:
is_Renamed.IndexReader.DeleteDocument(0, null);
hits = is_Renamed.Search(new MatchAllDocsQuery(), null, 1000, null).ScoreDocs;
Assert.AreEqual(2, hits.Length);
// test parsable toString()
QueryParser qp = new QueryParser(Util.Version.LUCENE_CURRENT, "key", analyzer);
hits = is_Renamed.Search(qp.Parse(new MatchAllDocsQuery().ToString()), null, 1000, null).ScoreDocs;
Assert.AreEqual(2, hits.Length);
// test parsable toString() with non default boost
Query maq = new MatchAllDocsQuery();
maq.Boost = 2.3f;
Query pq = qp.Parse(maq.ToString());
hits = is_Renamed.Search(pq, null, 1000, null).ScoreDocs;
Assert.AreEqual(2, hits.Length);
is_Renamed.Close();
ir.Close();
dir.Close();
}
public virtual void TestOpenPriorSnapshot()
{
// Never deletes a commit
KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(this);
Directory dir = new MockRAMDirectory();
policy.dir = dir;
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(2);
for (int i = 0; i < 10; i++)
{
AddDoc(writer);
if ((1 + i) % 2 == 0)
{
writer.Commit();
}
}
writer.Close();
System.Collections.ICollection commits = IndexReader.ListCommits(dir);
Assert.AreEqual(6, commits.Count);
IndexCommit lastCommit = null;
System.Collections.IEnumerator it = commits.GetEnumerator();
while (it.MoveNext())
{
IndexCommit commit = (IndexCommit)it.Current;
if (lastCommit == null || commit.GetGeneration() > lastCommit.GetGeneration())
{
lastCommit = commit;
}
}
Assert.IsTrue(lastCommit != null);
// Now add 1 doc and optimize
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED);
AddDoc(writer);
Assert.AreEqual(11, writer.NumDocs());
writer.Optimize();
writer.Close();
Assert.AreEqual(7, IndexReader.ListCommits(dir).Count);
// Now open writer on the commit just before optimize:
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED, lastCommit);
Assert.AreEqual(10, writer.NumDocs());
// Should undo our rollback:
writer.Rollback();
IndexReader r = IndexReader.Open(dir);
// Still optimized, still 11 docs
Assert.IsTrue(r.IsOptimized());
Assert.AreEqual(11, r.NumDocs());
r.Close();
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED, lastCommit);
Assert.AreEqual(10, writer.NumDocs());
// Commits the rollback:
writer.Close();
// Now 8 because we made another commit
Assert.AreEqual(8, IndexReader.ListCommits(dir).Count);
r = IndexReader.Open(dir);
// Not optimized because we rolled it back, and now only
// 10 docs
Assert.IsTrue(!r.IsOptimized());
Assert.AreEqual(10, r.NumDocs());
r.Close();
// Reoptimize
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), policy, IndexWriter.MaxFieldLength.LIMITED);
writer.Optimize();
writer.Close();
r = IndexReader.Open(dir);
Assert.IsTrue(r.IsOptimized());
Assert.AreEqual(10, r.NumDocs());
r.Close();
// Now open writer on the commit just before optimize,
// but this time keeping only the last commit:
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), new KeepOnlyLastCommitDeletionPolicy(), IndexWriter.MaxFieldLength.LIMITED, lastCommit);
Assert.AreEqual(10, writer.NumDocs());
// Reader still sees optimized index, because writer
// opened on the prior commit has not yet committed:
r = IndexReader.Open(dir);
Assert.IsTrue(r.IsOptimized());
Assert.AreEqual(10, r.NumDocs());
r.Close();
writer.Close();
// Now reader sees unoptimized index:
r = IndexReader.Open(dir);
Assert.IsTrue(!r.IsOptimized());
Assert.AreEqual(10, r.NumDocs());
r.Close();
dir.Close();
}
public virtual void TestDeletedDocs()
{
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 19; i++)
{
writer.AddDocument(doc);
}
writer.Close();
IndexReader reader = IndexReader.Open(dir);
reader.DeleteDocument(5);
reader.Close();
System.IO.MemoryStream bos = new System.IO.MemoryStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.SetInfoStream(new System.IO.StreamWriter(bos));
//checker.setInfoStream(System.out);
CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method();
if (indexStatus.clean == false)
{
System.Console.Out.WriteLine("CheckIndex failed");
char[] tmpChar;
byte[] tmpByte;
tmpByte = bos.GetBuffer();
tmpChar = new char[bos.Length];
System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length);
System.Console.Out.WriteLine(new System.String(tmpChar));
Assert.Fail();
}
CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus)indexStatus.segmentInfos[0];
Assert.IsTrue(seg.openReaderPassed);
Assert.IsNotNull(seg.diagnostics);
Assert.IsNotNull(seg.fieldNormStatus);
Assert.IsNull(seg.fieldNormStatus.error);
Assert.AreEqual(1, seg.fieldNormStatus.totFields);
Assert.IsNotNull(seg.termIndexStatus);
Assert.IsNull(seg.termIndexStatus.error);
Assert.AreEqual(1, seg.termIndexStatus.termCount);
Assert.AreEqual(19, seg.termIndexStatus.totFreq);
Assert.AreEqual(18, seg.termIndexStatus.totPos);
Assert.IsNotNull(seg.storedFieldStatus);
Assert.IsNull(seg.storedFieldStatus.error);
Assert.AreEqual(18, seg.storedFieldStatus.docCount);
Assert.AreEqual(18, seg.storedFieldStatus.totFields);
Assert.IsNotNull(seg.termVectorStatus);
Assert.IsNull(seg.termVectorStatus.error);
Assert.AreEqual(18, seg.termVectorStatus.docCount);
Assert.AreEqual(18, seg.termVectorStatus.totVectors);
Assert.IsTrue(seg.diagnostics.Count > 0);
System.Collections.IList onlySegments = new System.Collections.ArrayList();
onlySegments.Add("_0");
Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true);
}
public virtual void TestKeepLastNDeletionPolicy()
{
int N = 5;
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
Directory dir = new RAMDirectory();
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
for (int j = 0; j < N + 1; j++)
{
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
for (int i = 0; i < 17; i++)
{
AddDoc(writer);
}
writer.Optimize();
writer.Close();
}
Assert.IsTrue(policy.numDelete > 0);
Assert.AreEqual(N + 1, policy.numOnInit);
if (autoCommit)
{
Assert.IsTrue(policy.numOnCommit > 1);
}
else
{
Assert.AreEqual(N + 1, policy.numOnCommit);
}
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = IndexReader.Open(dir);
reader.Close();
if (i == N)
{
Assert.Fail("should have failed on commits prior to last " + N);
}
}
catch (System.IO.IOException e)
{
if (i != N)
{
throw e;
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Close();
}
}
public virtual void TestDeleteAllNRT()
{
Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, false, new WhitespaceAnalyzer(), true);
modifier.SetMaxBufferedDocs(2);
modifier.SetMaxBufferedDeleteTerms(2);
int id = 0;
int value_Renamed = 100;
for (int i = 0; i < 7; i++)
{
AddDoc(modifier, ++id, value_Renamed);
}
modifier.Commit();
IndexReader reader = modifier.GetReader();
Assert.AreEqual(7, reader.NumDocs());
reader.Close();
AddDoc(modifier, ++id, value_Renamed);
AddDoc(modifier, ++id, value_Renamed);
// Delete all
modifier.DeleteAll();
reader = modifier.GetReader();
Assert.AreEqual(0, reader.NumDocs());
reader.Close();
// Roll it back
modifier.Rollback();
modifier.Close();
// Validate that the docs are still there
reader = IndexReader.Open(dir);
Assert.AreEqual(7, reader.NumDocs());
reader.Close();
dir.Close();
}
/// <summary> Make sure if modifier tries to commit but hits disk full that modifier
/// remains consistent and usable. Similar to TestIndexReader.testDiskFull().
/// </summary>
private void TestOperationsOnDiskFull(bool updates)
{
bool debug = false;
Term searchTerm = new Term("content", "aaa");
int START_COUNT = 157;
int END_COUNT = 144;
// First build up a starting index:
MockRAMDirectory startDir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(startDir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED, null);
for (int i = 0; i < 157; i++)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
d.Add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(d, null);
}
writer.Close();
long diskUsage = startDir.SizeInBytes();
long diskFree = diskUsage + 10;
System.IO.IOException err = null;
bool done = false;
// Iterate w/ ever increasing free disk space:
while (!done)
{
MockRAMDirectory dir = new MockRAMDirectory(startDir);
dir.SetPreventDoubleWrite(false);
IndexWriter modifier = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED, null);
modifier.SetMaxBufferedDocs(1000); // use flush or close
modifier.SetMaxBufferedDeleteTerms(1000); // use flush or close
// For each disk size, first try to commit against
// dir that will hit random IOExceptions & disk
// full; after, give it infinite disk space & turn
// off random IOExceptions & retry w/ same reader:
bool success = false;
for (int x = 0; x < 2; x++)
{
double rate = 0.1;
double diskRatio = ((double)diskFree) / diskUsage;
long thisDiskFree;
System.String testName;
if (0 == x)
{
thisDiskFree = diskFree;
if (diskRatio >= 2.0)
{
rate /= 2;
}
if (diskRatio >= 4.0)
{
rate /= 2;
}
if (diskRatio >= 6.0)
{
rate = 0.0;
}
if (debug)
{
System.Console.Out.WriteLine("\ncycle: " + diskFree + " bytes");
}
testName = "disk full during reader.close() @ " + thisDiskFree + " bytes";
}
else
{
thisDiskFree = 0;
rate = 0.0;
if (debug)
{
System.Console.Out.WriteLine("\ncycle: same writer: unlimited disk space");
}
testName = "reader re-use after disk full";
}
dir.SetMaxSizeInBytes(thisDiskFree);
dir.SetRandomIOExceptionRate(rate, diskFree);
try
{
if (0 == x)
{
int docId = 12;
for (int i = 0; i < 13; i++)
{
if (updates)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES,
Field.Index.NOT_ANALYZED));
d.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED));
modifier.UpdateDocument(new Term("id", System.Convert.ToString(docId)), d, null);
}
else
{
// deletes
modifier.DeleteDocuments(null, new Term("id", System.Convert.ToString(docId)));
// modifier.setNorm(docId, "contents", (float)2.0);
}
docId += 12;
}
}
modifier.Close();
success = true;
if (0 == x)
{
done = true;
}
}
catch (System.IO.IOException e)
{
if (debug)
{
System.Console.Out.WriteLine(" hit IOException: " + e);
System.Console.Out.WriteLine(e.StackTrace);
}
err = e;
if (1 == x)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + " hit IOException after disk space was freed up");
}
}
// If the close() succeeded, make sure there are
// no unreferenced files.
if (success)
{
Lucene.Net.Util._TestUtil.CheckIndex(dir);
TestIndexWriter.AssertNoUnreferencedFiles(dir, "after writer.close");
}
// Finally, verify index is not corrupt, and, if
// we succeeded, we see all docs changed, and if
// we failed, we see either all docs or no docs
// changed (transactional semantics):
IndexReader newReader = null;
try
{
newReader = IndexReader.Open((Directory)dir, true, null);
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + ":exception when creating IndexReader after disk full during close: " + e);
}
IndexSearcher searcher = new IndexSearcher(newReader);
ScoreDoc[] hits = null;
try
{
hits = searcher.Search(new TermQuery(searchTerm), null, 1000, null).ScoreDocs;
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + ": exception when searching: " + e);
}
int result2 = hits.Length;
if (success)
{
if (x == 0 && result2 != END_COUNT)
{
Assert.Fail(testName +
": method did not throw exception but hits.length for search on term 'aaa' is " +
result2 + " instead of expected " + END_COUNT);
}
else if (x == 1 && result2 != START_COUNT && result2 != END_COUNT)
{
// It's possible that the first exception was
// "recoverable" wrt pending deletes, in which
// case the pending deletes are retained and
// then re-flushing (with plenty of disk
// space) will succeed in flushing the
// deletes:
Assert.Fail(testName +
": method did not throw exception but hits.length for search on term 'aaa' is " +
result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
else
{
// On hitting exception we still may have added
// all docs:
if (result2 != START_COUNT && result2 != END_COUNT)
{
System.Console.Error.WriteLine(err.StackTrace);
Assert.Fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " +
result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
searcher.Close();
newReader.Close();
if (result2 == END_COUNT)
{
break;
}
}
dir.Close();
// Try again with 10 more bytes of free space:
diskFree += 10;
}
}
/// <summary>
/// Retrieve a read/write <see cref="IndexWriter" />
/// </summary>
/// <param name="provider"></param>
/// <param name="entity"></param>
/// <param name="modificationOperation"></param>
/// <returns></returns>
public IndexWriter GetIndexWriter(IDirectoryProvider provider, System.Type entity, bool modificationOperation)
{
// Have to close the reader before the writer is accessed.
IndexReader reader;
readers.TryGetValue(provider, out reader);
if (reader != null)
{
try
{
reader.Close();
}
catch (IOException ex)
{
throw new SearchException("Exception while closing IndexReader", ex);
}
finally
{
readers.Remove(provider);
// PH - Moved the exit lock out of the try otherwise it won't take place when we have an error closing the reader.
// Exit Lock added by Kailuo Wang, because the lock needs to be obtained immediately afterwards
object syncLock = searchFactoryImplementor.GetLockableDirectoryProviders()[provider];
Monitor.Exit(syncLock);
}
}
if (writers.ContainsKey(provider))
{
return writers[provider];
}
LockProvider(provider);
if (modificationOperation) dpStatistics[provider].Operations++;
try
{
Analyzer analyzer = entity != null
? searchFactoryImplementor.DocumentBuilders[entity].Analyzer
: new SimpleAnalyzer();
IndexWriter writer = new IndexWriter(provider.Directory, analyzer, false);
LuceneIndexingParameters indexingParams = searchFactoryImplementor.GetIndexingParameters(provider);
if (IsBatch)
{
writer.SetMergeFactor(indexingParams.BatchMergeFactor);
writer.SetMaxMergeDocs(indexingParams.BatchMaxMergeDocs);
writer.SetMaxBufferedDocs(indexingParams.BatchMaxBufferedDocs);
}
else
{
writer.SetMergeFactor(indexingParams.TransactionMergeFactor);
writer.SetMaxMergeDocs(indexingParams.TransactionMaxMergeDocs);
writer.SetMaxBufferedDocs(indexingParams.TransactionMaxBufferedDocs);
}
writers.Add(provider, writer);
return writer;
}
catch (IOException ex)
{
CleanUp(new SearchException("Unable to open IndexWriter" + (entity != null ? " for " + entity : ""), ex));
}
return null;
}
/// <summary> Make sure if modifier tries to commit but hits disk full that modifier
/// remains consistent and usable. Similar to TestIndexReader.testDiskFull().
/// </summary>
private void TestOperationsOnDiskFull(bool updates)
{
bool debug = false;
Term searchTerm = new Term("content", "aaa");
int START_COUNT = 157;
int END_COUNT = 144;
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = (0 == pass);
// First build up a starting index:
MockRAMDirectory startDir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(startDir, autoCommit, new WhitespaceAnalyzer(), true);
for (int i = 0; i < 157; i++)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
d.Add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(d);
}
writer.Close();
long diskUsage = startDir.SizeInBytes();
long diskFree = diskUsage + 10;
System.IO.IOException err = null;
bool done = false;
// Iterate w/ ever increasing free disk space:
while (!done)
{
MockRAMDirectory dir = new MockRAMDirectory(startDir);
dir.SetPreventDoubleWrite(false);
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
modifier.SetMaxBufferedDocs(1000); // use flush or close
modifier.SetMaxBufferedDeleteTerms(1000); // use flush or close
// For each disk size, first try to commit against
// dir that will hit random IOExceptions & disk
// full; after, give it infinite disk space & turn
// off random IOExceptions & retry w/ same reader:
bool success = false;
for (int x = 0; x < 2; x++)
{
double rate = 0.1;
double diskRatio = ((double) diskFree) / diskUsage;
long thisDiskFree;
System.String testName;
if (0 == x)
{
thisDiskFree = diskFree;
if (diskRatio >= 2.0)
{
rate /= 2;
}
if (diskRatio >= 4.0)
{
rate /= 2;
}
if (diskRatio >= 6.0)
{
rate = 0.0;
}
if (debug)
{
System.Console.Out.WriteLine("\ncycle: " + diskFree + " bytes");
}
testName = "disk full during reader.close() @ " + thisDiskFree + " bytes";
}
else
{
thisDiskFree = 0;
rate = 0.0;
if (debug)
{
System.Console.Out.WriteLine("\ncycle: same writer: unlimited disk space");
}
testName = "reader re-use after disk full";
}
dir.SetMaxSizeInBytes(thisDiskFree);
dir.SetRandomIOExceptionRate(rate, diskFree);
try
{
if (0 == x)
{
int docId = 12;
for (int i = 0; i < 13; i++)
{
if (updates)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
d.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED));
modifier.UpdateDocument(new Term("id", System.Convert.ToString(docId)), d);
}
else
{
// deletes
modifier.DeleteDocuments(new Term("id", System.Convert.ToString(docId)));
// modifier.setNorm(docId, "contents", (float)2.0);
}
docId += 12;
}
}
modifier.Close();
success = true;
if (0 == x)
{
done = true;
}
}
catch (System.IO.IOException e)
{
if (debug)
{
System.Console.Out.WriteLine(" hit IOException: " + e);
System.Console.Out.WriteLine(e.StackTrace);
}
err = e;
if (1 == x)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + " hit IOException after disk space was freed up");
}
}
// If the close() succeeded, make sure there are
// no unreferenced files.
if (success)
{
Lucene.Net.Util._TestUtil.CheckIndex(dir);
TestIndexWriter.AssertNoUnreferencedFiles(dir, "after writer.close");
}
// Finally, verify index is not corrupt, and, if
// we succeeded, we see all docs changed, and if
// we failed, we see either all docs or no docs
// changed (transactional semantics):
IndexReader newReader = null;
try
{
newReader = IndexReader.Open(dir);
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + ":exception when creating IndexReader after disk full during close: " + e);
}
IndexSearcher searcher = new IndexSearcher(newReader);
ScoreDoc[] hits = null;
try
{
hits = searcher.Search(new TermQuery(searchTerm), null, 1000).ScoreDocs;
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.Fail(testName + ": exception when searching: " + e);
}
int result2 = hits.Length;
if (success)
{
if (x == 0 && result2 != END_COUNT)
{
Assert.Fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + END_COUNT);
}
else if (x == 1 && result2 != START_COUNT && result2 != END_COUNT)
{
// It's possible that the first exception was
// "recoverable" wrt pending deletes, in which
// case the pending deletes are retained and
// then re-flushing (with plenty of disk
// space) will succeed in flushing the
// deletes:
Assert.Fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
else
{
// On hitting exception we still may have added
// all docs:
if (result2 != START_COUNT && result2 != END_COUNT)
{
System.Console.Error.WriteLine(err.StackTrace);
Assert.Fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
}
}
searcher.Close();
newReader.Close();
if (result2 == END_COUNT)
{
break;
}
}
dir.Close();
// Try again with 10 more bytes of free space:
diskFree += 10;
}
}
}
