public virtual void TestSetBufferSize()
{
System.IO.DirectoryInfo indexDir = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "testSetBufferSize"));
MockFSDirectory dir = new MockFSDirectory(indexDir, NewRandom());
try
{
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);
writer.UseCompoundFile = false;
for (int i = 0; i < 37; i++)
{
Document doc = new Document();
doc.Add(new Field("content", "aaa bbb ccc ddd" + i, Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field("id", "" + i, Field.Store.YES, Field.Index.ANALYZED));
writer.AddDocument(doc, null);
}
writer.Close();
dir.allIndexInputs.Clear();
IndexReader reader = IndexReader.Open((Directory)dir, false, null);
Term aaa = new Term("content", "aaa");
Term bbb = new Term("content", "bbb");
Term ccc = new Term("content", "ccc");
Assert.AreEqual(37, reader.DocFreq(ccc, null));
reader.DeleteDocument(0, null);
Assert.AreEqual(37, reader.DocFreq(aaa, null));
dir.tweakBufferSizes();
reader.DeleteDocument(4, null);
Assert.AreEqual(reader.DocFreq(bbb, null), 37);
dir.tweakBufferSizes();
IndexSearcher searcher = new IndexSearcher(reader);
ScoreDoc[] hits = searcher.Search(new TermQuery(bbb), null, 1000, null).ScoreDocs;
dir.tweakBufferSizes();
Assert.AreEqual(35, hits.Length);
dir.tweakBufferSizes();
hits = searcher.Search(new TermQuery(new Term("id", "33")), null, 1000, null).ScoreDocs;
dir.tweakBufferSizes();
Assert.AreEqual(1, hits.Length);
hits = searcher.Search(new TermQuery(aaa), null, 1000, null).ScoreDocs;
dir.tweakBufferSizes();
Assert.AreEqual(35, hits.Length);
searcher.Close();
reader.Close();
}
finally
{
_TestUtil.RmDir(indexDir);
}
}
public virtual void TestMoreMerges()
{
// main directory
Directory dir = new RAMDirectory();
// auxiliary directory
Directory aux = new RAMDirectory();
Directory aux2 = new RAMDirectory();
SetUpDirs(dir, aux);
IndexWriter writer = NewWriter(aux2, true);
writer.SetMaxBufferedDocs(100);
writer.SetMergeFactor(10);
writer.AddIndexesNoOptimize(new Directory[] { aux });
Assert.AreEqual(30, writer.DocCount());
Assert.AreEqual(3, writer.GetSegmentCount());
writer.Close();
IndexReader reader = IndexReader.Open(aux);
for (int i = 0; i < 27; i++)
{
reader.DeleteDocument(i);
}
Assert.AreEqual(3, reader.NumDocs());
reader.Close();
reader = IndexReader.Open(aux2);
for (int i = 0; i < 8; i++)
{
reader.DeleteDocument(i);
}
Assert.AreEqual(22, reader.NumDocs());
reader.Close();
writer = NewWriter(dir, false);
writer.SetMaxBufferedDocs(6);
writer.SetMergeFactor(4);
writer.AddIndexesNoOptimize(new Directory[] { aux, aux2 });
Assert.AreEqual(1025, writer.DocCount());
Assert.AreEqual(1000, writer.GetDocCount(0));
writer.Close();
// make sure the index is correct
VerifyNumDocs(dir, 1025);
}
public virtual void TestMergeAfterCopy()
{
// main directory
Directory dir = new RAMDirectory();
// auxiliary directory
Directory aux = new RAMDirectory();
SetUpDirs(dir, aux);
IndexReader reader = IndexReader.Open(aux);
for (int i = 0; i < 20; i++)
{
reader.DeleteDocument(i);
}
Assert.AreEqual(10, reader.NumDocs());
reader.Close();
IndexWriter writer = NewWriter(dir, false);
writer.SetMaxBufferedDocs(4);
writer.SetMergeFactor(4);
writer.AddIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) });
Assert.AreEqual(1020, writer.DocCount());
Assert.AreEqual(1000, writer.GetDocCount(0));
writer.Close();
// make sure the index is correct
VerifyNumDocs(dir, 1020);
}
/// <summary> 1. Get a norm from the original reader 2. Clone the original reader 3.
/// Delete a document and set the norm of the cloned reader 4. Verify the norms
/// are not the same on each reader 5. Verify the doc deleted is only in the
/// cloned reader 6. Try to delete a document in the original reader, an
/// exception should be thrown
///
/// </summary>
/// <param name="r1">IndexReader to perform tests on
/// </param>
/// <throws> Exception </throws>
private void PerformDefaultTests(IndexReader r1)
{
float norm1 = Similarity.DecodeNorm(r1.Norms("field1")[4]);
IndexReader pr1Clone = (IndexReader)r1.Clone();
pr1Clone.DeleteDocument(10);
pr1Clone.SetNorm(4, "field1", 0.5f);
Assert.IsTrue(Similarity.DecodeNorm(r1.Norms("field1")[4]) == norm1);
Assert.IsTrue(Similarity.DecodeNorm(pr1Clone.Norms("field1")[4]) != norm1);
Assert.IsTrue(!r1.IsDeleted(10));
Assert.IsTrue(pr1Clone.IsDeleted(10));
// try to update the original reader, which should throw an exception
try
{
r1.DeleteDocument(11);
Assert.Fail("Tried to delete doc 11 and an exception should have been thrown");
}
catch (System.Exception exception)
{
// expectted
}
pr1Clone.Close();
}
/// <summary> Deletes the document numbered <code>docNum</code>.</summary>
/// <seealso cref="IndexReader#DeleteDocument(int)">
/// </seealso>
/// <throws> IllegalStateException if the index is closed </throws>
public virtual void DeleteDocument(int docNum)
{
lock (directory)
{
AssureOpen();
CreateIndexReader();
indexReader.DeleteDocument(docNum);
}
}
public virtual void DoTestUndeleteAll()
{
sis.Read(dir);
IndexReader reader = OpenReader();
Assert.IsTrue(reader != null);
Assert.AreEqual(2, reader.NumDocs());
reader.DeleteDocument(0);
Assert.AreEqual(1, reader.NumDocs());
reader.UndeleteAll();
Assert.AreEqual(2, reader.NumDocs());
// Ensure undeleteAll survives commit/close/reopen:
reader.Commit();
reader.Close();
if (reader is MultiReader)
{
// MultiReader does not "own" the directory so it does
// not write the changes to sis on commit:
sis.Commit(dir);
}
sis.Read(dir);
reader = OpenReader();
Assert.AreEqual(2, reader.NumDocs());
reader.DeleteDocument(0);
Assert.AreEqual(1, reader.NumDocs());
reader.Commit();
reader.Close();
if (reader is MultiReader)
{
// MultiReader does not "own" the directory so it does
// not write the changes to sis on commit:
sis.Commit(dir);
}
sis.Read(dir);
reader = OpenReader();
Assert.AreEqual(1, reader.NumDocs());
}
private static bool DeleteWorked(int doc, IndexReader r)
{
bool exception = false;
try
{
// trying to delete from the original reader should throw an exception
r.DeleteDocument(doc);
}
catch (System.Exception ex)
{
exception = true;
}
return(!exception);
}
public virtual void TestBinaryFieldInIndex()
{
IFieldable binaryFldStored = new Field("binaryStored", System.Text.UTF8Encoding.UTF8.GetBytes(binaryValStored), Field.Store.YES);
IFieldable stringFldStored = new Field("stringStored", binaryValStored, Field.Store.YES, Field.Index.NO, Field.TermVector.NO);
// binary fields with store off are not allowed
Assert.Throws <ArgumentException>(
() => new Field("fail", System.Text.Encoding.UTF8.GetBytes(binaryValStored), Field.Store.NO));
Document doc = new Document();
doc.Add(binaryFldStored);
doc.Add(stringFldStored);
/* test for field count */
Assert.AreEqual(2, doc.fields_ForNUnit.Count);
/* add the doc to a ram index */
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED, null);
writer.AddDocument(doc, null);
writer.Close();
/* open a reader and fetch the document */
IndexReader reader = IndexReader.Open((Directory)dir, false, null);
Document docFromReader = reader.Document(0, null);
Assert.IsTrue(docFromReader != null);
/* fetch the binary stored field and compare it's content with the original one */
System.String binaryFldStoredTest = new System.String(System.Text.UTF8Encoding.UTF8.GetChars(docFromReader.GetBinaryValue("binaryStored", null)));
Assert.IsTrue(binaryFldStoredTest.Equals(binaryValStored));
/* fetch the string field and compare it's content with the original one */
System.String stringFldStoredTest = docFromReader.Get("stringStored", null);
Assert.IsTrue(stringFldStoredTest.Equals(binaryValStored));
/* delete the document from index */
reader.DeleteDocument(0, null);
Assert.AreEqual(0, reader.NumDocs());
reader.Close();
dir.Close();
}
} // doIndex
private static void removeAllDuplicateAndDeletedFiles(IndexableFileInfo[] fileInfos, string LuceneIndexDir, IndexCreationMode indexCreationMode)
{
if (indexCreationMode != IndexCreationMode.AppendToExistingIndex)
{
return;
}
IndexReader reader = IndexReader.Open(LuceneIndexDir);
try
{
int numDocs = reader.NumDocs();
for (int i = 0; i < numDocs; i++)
{
Document docToCheck = reader.Document(i);
bool removeDocFromIndex = true;
string filenameField = docToCheck.GetField("filename").StringValue();
string lastModified = (docToCheck.GetField("LastModified").StringValue());
foreach (IndexableFileInfo fi in fileInfos)
{
if (String.Compare(fi.Filename, filenameField, true) == 0 && DateTools.DateToString(fi.LastModified, DateTools.Resolution.SECOND) == lastModified)
{
removeDocFromIndex = false;
break;
}
} // foreach
if (removeDocFromIndex)
{
reader.DeleteDocument(i);
if (!reader.HasDeletions())
{
throw new Exception("error: deletion failed!!");
}
}
} // for each lucene doc
}
finally
{
reader.Close();
}
LuceneIndexer indexer = new LuceneIndexer(LuceneIndexDir, indexCreationMode); // open up the index again
indexer.CloseIndexWriter(OptimizeMode.DoOptimization); // just to optimize the index (which removes deleted items).
}
/// <summary> 1. Get a norm from the original reader 2. Clone the original reader 3.
/// Delete a document and set the norm of the cloned reader 4. Verify the norms
/// are not the same on each reader 5. Verify the doc deleted is only in the
/// cloned reader 6. Try to delete a document in the original reader, an
/// exception should be thrown
///
/// </summary>
/// <param name="r1">IndexReader to perform tests on
/// </param>
/// <throws> Exception </throws>
private void PerformDefaultTests(IndexReader r1)
{
float norm1 = Similarity.DecodeNorm(r1.Norms("field1")[4]);
IndexReader pr1Clone = (IndexReader)r1.Clone();
pr1Clone.DeleteDocument(10);
pr1Clone.SetNorm(4, "field1", 0.5f);
Assert.IsTrue(Similarity.DecodeNorm(r1.Norms("field1")[4]) == norm1);
Assert.IsTrue(Similarity.DecodeNorm(pr1Clone.Norms("field1")[4]) != norm1);
Assert.IsTrue(!r1.IsDeleted(10));
Assert.IsTrue(pr1Clone.IsDeleted(10));
// try to update the original reader, which should throw an exception
Assert.Throws <LockObtainFailedException>(() => r1.DeleteDocument(11),
"Tried to delete doc 11 and an exception should have been thrown");
pr1Clone.Close();
}
public virtual void TestCloneWithDeletes()
{
Directory dir1 = new MockRAMDirectory();
TestIndexReaderReopen.CreateIndex(dir1, false);
IndexReader origReader = IndexReader.Open(dir1, false);
origReader.DeleteDocument(1);
IndexReader clonedReader = (IndexReader)origReader.Clone();
origReader.Close();
clonedReader.Close();
IndexReader r = IndexReader.Open(dir1, false);
Assert.IsTrue(r.IsDeleted(1));
r.Close();
dir1.Close();
}
public virtual void TestLucene1516Bug()
{
Directory dir1 = new MockRAMDirectory();
TestIndexReaderReopen.CreateIndex(dir1, false);
IndexReader r1 = IndexReader.Open(dir1, false);
r1.IncRef();
IndexReader r2 = r1.Clone(false);
r1.DeleteDocument(5);
r1.DecRef();
r1.IncRef();
r2.Close();
r1.DecRef();
r1.Close();
dir1.Close();
}
public virtual void TestCrashReaderDeletes()
{
IndexWriter writer = InitIndex();
MockRAMDirectory dir = (MockRAMDirectory)writer.Directory;
writer.Close(false);
IndexReader reader = IndexReader.Open((Directory)dir, false, null);
reader.DeleteDocument(3, null);
dir.Crash();
/*
* String[] l = dir.list();
* Arrays.sort(l);
* for(int i=0;i<l.length;i++)
* System.out.println("file " + i + " = " + l[i] + " " + dir.fileLength(l[i]) + " bytes");
*/
reader = IndexReader.Open((Directory)dir, false, null);
Assert.AreEqual(157, reader.NumDocs());
}
public virtual void TestCloneSubreaders()
{
Directory dir1 = new MockRAMDirectory();
TestIndexReaderReopen.CreateIndex(dir1, true);
IndexReader reader = IndexReader.Open(dir1, false);
reader.DeleteDocument(1); // acquire write lock
IndexReader[] subs = reader.GetSequentialSubReaders();
System.Diagnostics.Debug.Assert(subs.Length > 1);
IndexReader[] clones = new IndexReader[subs.Length];
for (int x = 0; x < subs.Length; x++)
{
clones[x] = (IndexReader)subs[x].Clone();
}
reader.Close();
for (int x = 0; x < subs.Length; x++)
{
clones[x].Close();
}
dir1.Close();
}
// Apply buffered delete terms to the segment just flushed from ram
// apply appropriately so that a delete term is only applied to
// the documents buffered before it, not those buffered after it.
private void ApplyDeletesSelectively(System.Collections.Hashtable deleteTerms, System.Collections.IList deleteIds, IndexReader reader)
{
System.Collections.IEnumerator iter = new System.Collections.Hashtable(deleteTerms).GetEnumerator();
while (iter.MoveNext())
{
System.Collections.DictionaryEntry entry = (System.Collections.DictionaryEntry) iter.Current;
Term term = (Term) entry.Key;
TermDocs docs = reader.TermDocs(term);
if (docs != null)
{
int num = ((DocumentsWriter.Num) entry.Value).GetNum();
try
{
while (docs.Next())
{
int doc = docs.Doc();
if (doc >= num)
{
break;
}
reader.DeleteDocument(doc);
}
}
finally
{
docs.Close();
}
}
}
if (deleteIds.Count > 0)
{
iter = deleteIds.GetEnumerator();
while (iter.MoveNext())
{
reader.DeleteDocument(((System.Int32) iter.Current));
}
}
}
public virtual void TestKeepLastNDeletionPolicyWithCreates()
{
int N = 10;
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
writer.Close();
Term searchTerm = new Term("content", "aaa");
Query query = new TermQuery(searchTerm);
for (int i = 0; i < N + 1; i++)
{
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
for (int j = 0; j < 17; j++)
{
AddDoc(writer);
}
// this is a commit when autoCommit=false:
writer.Close();
IndexReader reader = IndexReader.Open(dir, policy);
reader.DeleteDocument(3);
reader.SetNorm(5, "content", 2.0F);
IndexSearcher searcher = new IndexSearcher(reader);
ScoreDoc[] hits = searcher.Search(query, null, 1000).scoreDocs;
Assert.AreEqual(16, hits.Length);
// this is a commit when autoCommit=false:
reader.Close();
searcher.Close();
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
// This will not commit: there are no changes
// pending because we opened for "create":
writer.Close();
}
Assert.AreEqual(1 + 3 * (N + 1), policy.numOnInit);
if (!autoCommit)
{
Assert.AreEqual(3 * (N + 1), policy.numOnCommit);
}
IndexSearcher searcher2 = new IndexSearcher(dir);
ScoreDoc[] hits2 = searcher2.Search(query, null, 1000).scoreDocs;
Assert.AreEqual(0, hits2.Length);
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
int expectedCount = 0;
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = IndexReader.Open(dir);
// Work backwards in commits on what the expected
// count should be. Only check this in the
// autoCommit false case:
if (!autoCommit)
{
searcher2 = new IndexSearcher(reader);
hits2 = searcher2.Search(query, null, 1000).scoreDocs;
Assert.AreEqual(expectedCount, hits2.Length);
searcher2.Close();
if (expectedCount == 0)
{
expectedCount = 16;
}
else if (expectedCount == 16)
{
expectedCount = 17;
}
else if (expectedCount == 17)
{
expectedCount = 0;
}
}
reader.Close();
if (i == N)
{
Assert.Fail("should have failed on commits before last " + N);
}
}
catch (System.IO.IOException e)
{
if (i != N)
{
throw e;
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Close();
}
}
/// <summary> 1. Get a norm from the original reader 2. Clone the original reader 3.
/// Delete a document and set the norm of the cloned reader 4. Verify the norms
/// are not the same on each reader 5. Verify the doc deleted is only in the
/// cloned reader 6. Try to delete a document in the original reader, an
/// exception should be thrown
///
/// </summary>
/// <param name="r1">IndexReader to perform tests on
/// </param>
/// <throws> Exception </throws>
private void PerformDefaultTests(IndexReader r1)
{
float norm1 = Similarity.DecodeNorm(r1.Norms("field1")[4]);
IndexReader pr1Clone = (IndexReader) r1.Clone();
pr1Clone.DeleteDocument(10);
pr1Clone.SetNorm(4, "field1", 0.5f);
Assert.IsTrue(Similarity.DecodeNorm(r1.Norms("field1")[4]) == norm1);
Assert.IsTrue(Similarity.DecodeNorm(pr1Clone.Norms("field1")[4]) != norm1);
Assert.IsTrue(!r1.IsDeleted(10));
Assert.IsTrue(pr1Clone.IsDeleted(10));
// try to update the original reader, which should throw an exception
Assert.Throws<LockObtainFailedException>(() => r1.DeleteDocument(11),
"Tried to delete doc 11 and an exception should have been thrown");
pr1Clone.Close();
}
// Apply buffered delete terms, queries and docIDs to the
// provided reader
private bool ApplyDeletes(IndexReader reader, int docIDStart)
{
lock (this)
{
int docEnd = docIDStart + reader.MaxDoc();
bool any = false;
System.Diagnostics.Debug.Assert(CheckDeleteTerm(null));
// Delete by term
//System.Collections.IEnumerator iter = new System.Collections.Hashtable(deletesFlushed.terms).GetEnumerator();
System.Collections.IEnumerator iter = deletesFlushed.terms.GetEnumerator();
TermDocs docs = reader.TermDocs();
try
{
while (iter.MoveNext())
{
System.Collections.DictionaryEntry entry = (System.Collections.DictionaryEntry) iter.Current;
Term term = (Term) entry.Key;
// LUCENE-2086: we should be iterating a TreeMap,
// here, so terms better be in order:
System.Diagnostics.Debug.Assert(CheckDeleteTerm(term));
docs.Seek(term);
int limit = ((BufferedDeletes.Num) entry.Value).GetNum();
while (docs.Next())
{
int docID = docs.Doc();
if (docIDStart + docID >= limit)
break;
reader.DeleteDocument(docID);
any = true;
}
}
}
finally
{
docs.Close();
}
// Delete by docID
iter = deletesFlushed.docIDs.GetEnumerator();
while (iter.MoveNext())
{
int docID = ((System.Int32) iter.Current);
if (docID >= docIDStart && docID < docEnd)
{
reader.DeleteDocument(docID - docIDStart);
any = true;
}
}
// Delete by query
IndexSearcher searcher = new IndexSearcher(reader);
iter = new System.Collections.Hashtable(deletesFlushed.queries).GetEnumerator();
while (iter.MoveNext())
{
System.Collections.DictionaryEntry entry = (System.Collections.DictionaryEntry) iter.Current;
Query query = (Query) entry.Key;
int limit = ((System.Int32) entry.Value);
Weight weight = query.Weight(searcher);
Scorer scorer = weight.Scorer(reader, true, false);
if (scorer != null)
{
while (true)
{
int doc = scorer.NextDoc();
if (((long) docIDStart) + doc >= limit)
break;
reader.DeleteDocument(doc);
any = true;
}
}
}
searcher.Close();
return any;
}
}
protected internal override void DoDelete(int n, IState state)
{
in_Renamed.DeleteDocument(n, state);
}
private static bool DeleteWorked(int doc, IndexReader r)
{
bool exception = false;
try
{
// trying to delete from the original reader should throw an exception
r.DeleteDocument(doc);
}
catch (System.Exception ex)
{
exception = true;
}
return !exception;
}
public virtual void TestSegmentReaderDelDocsReferenceCounting()
{
Directory dir1 = new MockRAMDirectory();
TestIndexReaderReopen.CreateIndex(dir1, false);
IndexReader origReader = IndexReader.Open(dir1, false);
SegmentReader origSegmentReader = SegmentReader.GetOnlySegmentReader(origReader);
// deletedDocsRef should be null because nothing has updated yet
Assert.IsNull(origSegmentReader.deletedDocsRef_ForNUnit);
// we deleted a document, so there is now a deletedDocs bitvector and a
// reference to it
origReader.DeleteDocument(1);
AssertDelDocsRefCountEquals(1, origSegmentReader);
// the cloned segmentreader should have 2 references, 1 to itself, and 1 to
// the original segmentreader
IndexReader clonedReader = (IndexReader)origReader.Clone();
SegmentReader clonedSegmentReader = SegmentReader.GetOnlySegmentReader(clonedReader);
AssertDelDocsRefCountEquals(2, origSegmentReader);
// deleting a document creates a new deletedDocs bitvector, the refs goes to
// 1
clonedReader.DeleteDocument(2);
AssertDelDocsRefCountEquals(1, origSegmentReader);
AssertDelDocsRefCountEquals(1, clonedSegmentReader);
// make sure the deletedocs objects are different (copy
// on write)
Assert.IsTrue(origSegmentReader.deletedDocs_ForNUnit != clonedSegmentReader.deletedDocs_ForNUnit);
AssertDocDeleted(origSegmentReader, clonedSegmentReader, 1);
Assert.IsTrue(!origSegmentReader.IsDeleted(2)); // doc 2 should not be deleted
// in original segmentreader
Assert.IsTrue(clonedSegmentReader.IsDeleted(2)); // doc 2 should be deleted in
// cloned segmentreader
// deleting a doc from the original segmentreader should throw an exception
Assert.Throws <LockObtainFailedException>(() => origReader.DeleteDocument(4), "expected exception");
origReader.Close();
// try closing the original segment reader to see if it affects the
// clonedSegmentReader
clonedReader.DeleteDocument(3);
clonedReader.Flush();
AssertDelDocsRefCountEquals(1, clonedSegmentReader);
// test a reopened reader
IndexReader reopenedReader = clonedReader.Reopen();
IndexReader cloneReader2 = (IndexReader)reopenedReader.Clone();
SegmentReader cloneSegmentReader2 = SegmentReader.GetOnlySegmentReader(cloneReader2);
AssertDelDocsRefCountEquals(2, cloneSegmentReader2);
clonedReader.Close();
reopenedReader.Close();
cloneReader2.Close();
dir1.Close();
}
// Apply buffered delete terms, queries and docIDs to the
// provided reader
private bool ApplyDeletes(IndexReader reader, int docIDStart)
{
lock (this)
{
int docEnd = docIDStart + reader.MaxDoc();
bool any = false;
// Delete by term
IEnumerator<KeyValuePair<object, object>> iter = deletesFlushed.terms.GetEnumerator();
while (iter.MoveNext())
{
KeyValuePair<object, object> entry = (KeyValuePair<object, object>)iter.Current;
Term term = (Term)entry.Key;
TermDocs docs = reader.TermDocs(term);
if (docs != null)
{
int limit = ((BufferedDeletes.Num)entry.Value).GetNum();
try
{
while (docs.Next())
{
int docID = docs.Doc();
if (docIDStart + docID >= limit)
break;
reader.DeleteDocument(docID);
any = true;
}
}
finally
{
docs.Close();
}
}
}
// Delete by docID
IEnumerator<object> iter2 = deletesFlushed.docIDs.GetEnumerator();
while (iter2.MoveNext())
{
int docID = (int)iter2.Current;
if (docID >= docIDStart && docID < docEnd)
{
reader.DeleteDocument(docID - docIDStart);
any = true;
}
}
// Delete by query
IndexSearcher searcher = new IndexSearcher(reader);
iter = deletesFlushed.queries.GetEnumerator();
while (iter.MoveNext())
{
KeyValuePair<object, object> entry = (KeyValuePair<object, object>)iter.Current;
Query query = (Query)entry.Key;
int limit = (int)entry.Value;
Weight weight = query.Weight(searcher);
Scorer scorer = weight.Scorer(reader);
while (scorer.Next())
{
int docID = scorer.Doc();
if (docIDStart + docID >= limit)
break;
reader.DeleteDocument(docID);
any = true;
}
}
searcher.Close();
return any;
}
}
// Apply buffered delete terms, queries and docIDs to the
// provided reader
private bool ApplyDeletes(IndexReader reader, int docIDStart)
{
lock (this)
{
int docEnd = docIDStart + reader.MaxDoc();
bool any = false;
System.Diagnostics.Debug.Assert(CheckDeleteTerm(null));
// Delete by term
TermDocs docs = reader.TermDocs();
try
{
foreach(KeyValuePair<Term,BufferedDeletes.Num> entry in deletesFlushed.terms)
{
Term term = entry.Key;
// LUCENE-2086: we should be iterating a TreeMap,
// here, so terms better be in order:
System.Diagnostics.Debug.Assert(CheckDeleteTerm(term));
docs.Seek(term);
int limit = entry.Value.GetNum();
while (docs.Next())
{
int docID = docs.Doc();
if (docIDStart + docID >= limit)
break;
reader.DeleteDocument(docID);
any = true;
}
}
}
finally
{
docs.Close();
}
// Delete by docID
foreach(int docID in deletesFlushed.docIDs)
{
if (docID >= docIDStart && docID < docEnd)
{
reader.DeleteDocument(docID - docIDStart);
any = true;
}
}
// Delete by query
IndexSearcher searcher = new IndexSearcher(reader);
foreach(KeyValuePair<Query,int> entry in new Support.Dictionary<Query,int>(deletesFlushed.queries))
{
Query query = entry.Key;
int limit = entry.Value;
Weight weight = query.Weight(searcher);
Scorer scorer = weight.Scorer(reader, true, false);
if (scorer != null)
{
while (true)
{
int doc = scorer.NextDoc();
if (((long) docIDStart) + doc >= limit)
break;
reader.DeleteDocument(doc);
any = true;
}
}
}
searcher.Close();
return any;
}
}
public virtual void TestKeepLastNDeletionPolicyWithReader()
{
int N = 10;
for (int pass = 0; pass < 2; pass++)
{
bool useCompoundFile = (pass % 2) != 0;
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(this, N);
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, policy, IndexWriter.MaxFieldLength.UNLIMITED);
writer.UseCompoundFile = useCompoundFile;
writer.Close();
Term searchTerm = new Term("content", "aaa");
Query query = new TermQuery(searchTerm);
for (int i = 0; i < N + 1; i++)
{
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.UNLIMITED);
writer.UseCompoundFile = useCompoundFile;
for (int j = 0; j < 17; j++)
{
AddDoc(writer);
}
// this is a commit
writer.Close();
IndexReader reader = IndexReader.Open(dir, policy, false);
reader.DeleteDocument(3 * i + 1);
reader.SetNorm(4 * i + 1, "content", 2.0F);
IndexSearcher searcher = new IndexSearcher(reader);
ScoreDoc[] hits = searcher.Search(query, null, 1000).ScoreDocs;
Assert.AreEqual(16 * (1 + i), hits.Length);
// this is a commit
reader.Close();
searcher.Close();
}
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.UNLIMITED);
writer.UseCompoundFile = useCompoundFile;
writer.Optimize();
// this is a commit
writer.Close();
Assert.AreEqual(2 * (N + 2), policy.numOnInit);
Assert.AreEqual(2 * (N + 2) - 1, policy.numOnCommit);
IndexSearcher searcher2 = new IndexSearcher(dir, false);
ScoreDoc[] hits2 = searcher2.Search(query, null, 1000).ScoreDocs;
Assert.AreEqual(176, hits2.Length);
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
int expectedCount = 176;
for (int i = 0; i < N + 1; i++)
{
try
{
IndexReader reader = IndexReader.Open(dir, true);
// Work backwards in commits on what the expected
// count should be.
searcher2 = new IndexSearcher(reader);
hits2 = searcher2.Search(query, null, 1000).ScoreDocs;
if (i > 1)
{
if (i % 2 == 0)
{
expectedCount += 1;
}
else
{
expectedCount -= 17;
}
}
Assert.AreEqual(expectedCount, hits2.Length);
searcher2.Close();
reader.Close();
if (i == N)
{
Assert.Fail("should have failed on commits before last 5");
}
}
catch (System.IO.IOException e)
{
if (i != N)
{
throw e;
}
}
if (i < N)
{
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.Close();
}
}
protected internal override void DoDelete(int n)
{
in_Renamed.DeleteDocument(n);
}
public virtual void TestDeletedDocs()
{
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 19; i++)
{
writer.AddDocument(doc);
}
writer.Close();
IndexReader reader = IndexReader.Open(dir);
reader.DeleteDocument(5);
reader.Close();
System.IO.MemoryStream bos = new System.IO.MemoryStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.SetInfoStream(new System.IO.StreamWriter(bos));
//checker.setInfoStream(System.out);
CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method();
if (indexStatus.clean == false)
{
System.Console.Out.WriteLine("CheckIndex failed");
char[] tmpChar;
byte[] tmpByte;
tmpByte = bos.GetBuffer();
tmpChar = new char[bos.Length];
System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length);
System.Console.Out.WriteLine(new System.String(tmpChar));
Assert.Fail();
}
CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus)indexStatus.segmentInfos[0];
Assert.IsTrue(seg.openReaderPassed);
Assert.IsNotNull(seg.diagnostics);
Assert.IsNotNull(seg.fieldNormStatus);
Assert.IsNull(seg.fieldNormStatus.error);
Assert.AreEqual(1, seg.fieldNormStatus.totFields);
Assert.IsNotNull(seg.termIndexStatus);
Assert.IsNull(seg.termIndexStatus.error);
Assert.AreEqual(1, seg.termIndexStatus.termCount);
Assert.AreEqual(19, seg.termIndexStatus.totFreq);
Assert.AreEqual(18, seg.termIndexStatus.totPos);
Assert.IsNotNull(seg.storedFieldStatus);
Assert.IsNull(seg.storedFieldStatus.error);
Assert.AreEqual(18, seg.storedFieldStatus.docCount);
Assert.AreEqual(18, seg.storedFieldStatus.totFields);
Assert.IsNotNull(seg.termVectorStatus);
Assert.IsNull(seg.termVectorStatus.error);
Assert.AreEqual(18, seg.termVectorStatus.docCount);
Assert.AreEqual(18, seg.termVectorStatus.totVectors);
Assert.IsTrue(seg.diagnostics.Count > 0);
System.Collections.IList onlySegments = new System.Collections.ArrayList();
onlySegments.Add("_0");
Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true);
}
/// <summary> 1. Get a norm from the original reader 2. Clone the original reader 3.
/// Delete a document and set the norm of the cloned reader 4. Verify the norms
/// are not the same on each reader 5. Verify the doc deleted is only in the
/// cloned reader 6. Try to delete a document in the original reader, an
/// exception should be thrown
///
/// </summary>
/// <param name="r1">IndexReader to perform tests on
/// </param>
/// <throws> Exception </throws>
private void PerformDefaultTests(IndexReader r1)
{
float norm1 = Similarity.DecodeNorm(r1.Norms("field1")[4]);
IndexReader pr1Clone = (IndexReader) r1.Clone();
pr1Clone.DeleteDocument(10);
pr1Clone.SetNorm(4, "field1", 0.5f);
Assert.IsTrue(Similarity.DecodeNorm(r1.Norms("field1")[4]) == norm1);
Assert.IsTrue(Similarity.DecodeNorm(pr1Clone.Norms("field1")[4]) != norm1);
Assert.IsTrue(!r1.IsDeleted(10));
Assert.IsTrue(pr1Clone.IsDeleted(10));
// try to update the original reader, which should throw an exception
try
{
r1.DeleteDocument(11);
Assert.Fail("Tried to delete doc 11 and an exception should have been thrown");
}
catch (System.Exception exception)
{
// expectted
}
pr1Clone.Close();
}
