public override void TearDown()
{
Iw.Dispose();
TestUtil.CheckIndex(Dir); // for some extra coverage, checkIndex before we forceMerge
Iwc.SetOpenMode(IndexWriterConfig.OpenMode_e.APPEND);
IndexWriter iw = new IndexWriter(Dir, (IndexWriterConfig)Iwc.Clone());
iw.ForceMerge(1);
iw.Dispose();
Dir.Dispose(); // just force a checkindex for now
base.TearDown();
}
public virtual void SearchFiltered(IndexWriter writer, Directory directory, Filter filter, bool fullMerge)
{
for (int i = 0; i < 60; i++) //Simple docs
{
Document doc = new Document();
doc.Add(NewStringField(FIELD, Convert.ToString(i), Field.Store.YES));
writer.AddDocument(doc);
}
if (fullMerge)
{
writer.ForceMerge(1);
}
writer.Dispose();
BooleanQuery booleanQuery = new BooleanQuery();
booleanQuery.Add(new TermQuery(new Term(FIELD, "36")), BooleanClause.Occur.SHOULD);
IndexReader reader = DirectoryReader.Open(directory);
IndexSearcher indexSearcher = NewSearcher(reader);
ScoreDoc[] hits = indexSearcher.Search(booleanQuery, filter, 1000).ScoreDocs;
Assert.AreEqual(1, hits.Length, "Number of matched documents");
reader.Dispose();
}
public void TestCustomMergeScheduler()
{
// we don't really need to execute anything, just to make sure the custom MS
// compiles. But ensure that it can be used as well, e.g., no other hidden
// dependencies or something. Therefore, don't use any random API !
Directory dir = new RAMDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.SetMergeScheduler(new ReportingMergeScheduler());
IndexWriter writer = new IndexWriter(dir, conf);
writer.AddDocument(new Document());
writer.Commit(); // trigger flush
writer.AddDocument(new Document());
writer.Commit(); // trigger flush
writer.ForceMerge(1);
writer.Dispose();
dir.Dispose();
}
public virtual void TestOpenIfChangedNoChangesButSegmentMerges()
{
// test openIfChanged() when the taxonomy hasn't really changed, but segments
// were merged. The NRT reader will be reopened, and ParentArray used to assert
// that the new reader contains more ordinals than were given from the old
// TaxReader version
Directory dir = NewDirectory();
// hold onto IW to forceMerge
// note how we don't close it, since DTW will close it.
var iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(new LogByteSizeMergePolicy()));
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriterAnonymousInnerClassHelper3(this, dir, iw);
// add a category so that the following DTR open will cause a flush and
// a new segment will be created
writer.AddCategory(new FacetLabel("a"));
var reader = new DirectoryTaxonomyReader(writer);
Assert.AreEqual(2, reader.Size);
Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length);
// merge all the segments so that NRT reader thinks there's a change
iw.ForceMerge(1);
// now calling openIfChanged should trip on the wrong assert in ParetArray's ctor
var newtr = TaxonomyReader.OpenIfChanged(reader);
Assert.NotNull(newtr);
reader.Dispose();
reader = newtr;
Assert.AreEqual(2, reader.Size);
Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length);
reader.Dispose();
writer.Dispose();
dir.Dispose();
}
public virtual void TestOpenIfChangedMergedSegment()
{
// test openIfChanged() when all index segments were merged - used to be
// a bug in ParentArray, caught by testOpenIfChangedManySegments - only
// this test is not random
Directory dir = NewDirectory();
// hold onto IW to forceMerge
// note how we don't close it, since DTW will close it.
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(new LogByteSizeMergePolicy()));
var writer = new DirectoryTaxonomyWriterAnonymousInnerClassHelper2(this, dir, iw) as DirectoryTaxonomyWriter;
var reader = new DirectoryTaxonomyReader(writer);
Assert.AreEqual(1, reader.Size);
Assert.AreEqual(1, reader.ParallelTaxonomyArrays.Parents().Length);
// add category and call forceMerge -- this should flush IW and merge segments down to 1
// in ParentArray.initFromReader, this used to fail assuming there are no parents.
writer.AddCategory(new FacetLabel("1"));
iw.ForceMerge(1);
// now calling openIfChanged should trip on the bug
var newtr = TaxonomyReader.OpenIfChanged(reader);
Assert.NotNull(newtr);
reader.Dispose();
reader = newtr;
Assert.AreEqual(2, reader.Size);
Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length);
reader.Dispose();
writer.Dispose();
dir.Dispose();
}
public virtual void TestSparseIndex()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
for (int d = -20; d <= 20; d++)
{
Document doc = new Document();
doc.Add(NewStringField("id", Convert.ToString(d), Field.Store.NO));
doc.Add(NewStringField("body", "body", Field.Store.NO));
writer.AddDocument(doc);
}
writer.ForceMerge(1);
writer.DeleteDocuments(new Term("id", "0"));
writer.Dispose();
IndexReader reader = DirectoryReader.Open(dir);
IndexSearcher search = NewSearcher(reader);
Assert.IsTrue(reader.HasDeletions);
ScoreDoc[] result;
Query q = new TermQuery(new Term("body", "body"));
result = search.Search(q, FieldCacheRangeFilter.NewByteRange("id", (sbyte?)-20, (sbyte?)20, T, T), 100).ScoreDocs;
Assert.AreEqual(40, result.Length, "find all");
result = search.Search(q, FieldCacheRangeFilter.NewByteRange("id", (sbyte?)0, (sbyte?)20, T, T), 100).ScoreDocs;
Assert.AreEqual(20, result.Length, "find all");
result = search.Search(q, FieldCacheRangeFilter.NewByteRange("id", (sbyte?)-20, (sbyte?)0, T, T), 100).ScoreDocs;
Assert.AreEqual(20, result.Length, "find all");
result = search.Search(q, FieldCacheRangeFilter.NewByteRange("id", (sbyte?)10, (sbyte?)20, T, T), 100).ScoreDocs;
Assert.AreEqual(11, result.Length, "find all");
result = search.Search(q, FieldCacheRangeFilter.NewByteRange("id", (sbyte?)-20, (sbyte?)-10, T, T), 100).ScoreDocs;
Assert.AreEqual(11, result.Length, "find all");
reader.Dispose();
dir.Dispose();
}
private static IndexReader MakeEmptyIndex(Random random, int numDocs)
{
Debug.Assert(numDocs > 0);
Directory d = new MockDirectoryWrapper(random, new RAMDirectory());
IndexWriter w = new IndexWriter(d, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(random)));
for (int i = 0; i < numDocs; i++)
{
w.AddDocument(new Document());
}
w.ForceMerge(1);
w.Commit();
w.Dispose();
DirectoryReader reader = DirectoryReader.Open(d);
return new AllDeletedFilterReader(LuceneTestCase.GetOnlySegmentReader(reader));
}
public virtual void TestMultiValuedField()
{
Directory indexStore = NewDirectory();
IndexWriter writer = new IndexWriter(indexStore, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
for (int i = 0; i < 5; i++)
{
Document doc = new Document();
doc.Add(new StringField("string", "a" + i, Field.Store.NO));
doc.Add(new StringField("string", "b" + i, Field.Store.NO));
writer.AddDocument(doc);
}
writer.ForceMerge(1); // enforce one segment to have a higher unique term count in all cases
writer.Dispose();
Sort sort = new Sort(new SortField("string", SortField.Type_e.STRING), SortField.FIELD_DOC);
// this should not throw AIOOBE or RuntimeEx
IndexReader reader = DirectoryReader.Open(indexStore);
IndexSearcher searcher = NewSearcher(reader);
searcher.Search(new MatchAllDocsQuery(), null, 500, sort);
reader.Dispose();
indexStore.Dispose();
}
// Test using various international locales with accented characters (which
// sort differently depending on locale)
//
// Copied (and slightly modified) from
// Lucene.Net.Search.TestSort.testInternationalSort()
//
// TODO: this test is really fragile. there are already 3 different cases,
// depending upon unicode version.
public virtual void TestCollationKeySort(Analyzer usAnalyzer, Analyzer franceAnalyzer, Analyzer swedenAnalyzer, Analyzer denmarkAnalyzer, string usResult, string frResult, string svResult, string dkResult)
{
Directory indexStore = NewDirectory();
IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false)));
// document data:
// the tracer field is used to determine which document was hit
string[][] sortData = new string[][] { new string[] { "A", "x", "p\u00EAche", "p\u00EAche", "p\u00EAche", "p\u00EAche" }, new string[] { "B", "y", "HAT", "HAT", "HAT", "HAT" }, new string[] { "C", "x", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9" }, new string[] { "D", "y", "HUT", "HUT", "HUT", "HUT" }, new string[] { "E", "x", "peach", "peach", "peach", "peach" }, new string[] { "F", "y", "H\u00C5T", "H\u00C5T", "H\u00C5T", "H\u00C5T" }, new string[] { "G", "x", "sin", "sin", "sin", "sin" }, new string[] { "H", "y", "H\u00D8T", "H\u00D8T", "H\u00D8T", "H\u00D8T" }, new string[] { "I", "x", "s\u00EDn", "s\u00EDn", "s\u00EDn", "s\u00EDn" }, new string[] { "J", "y", "HOT", "HOT", "HOT", "HOT" } };
FieldType customType = new FieldType();
customType.Stored = true;
for (int i = 0; i < sortData.Length; ++i)
{
Document doc = new Document();
doc.Add(new Field("tracer", sortData[i][0], customType));
doc.Add(new TextField("contents", sortData[i][1], Field.Store.NO));
if (sortData[i][2] != null)
{
doc.Add(new TextField("US", usAnalyzer.TokenStream("US", new StringReader(sortData[i][2]))));
}
if (sortData[i][3] != null)
{
doc.Add(new TextField("France", franceAnalyzer.TokenStream("France", new StringReader(sortData[i][3]))));
}
if (sortData[i][4] != null)
{
doc.Add(new TextField("Sweden", swedenAnalyzer.TokenStream("Sweden", new StringReader(sortData[i][4]))));
}
if (sortData[i][5] != null)
{
doc.Add(new TextField("Denmark", denmarkAnalyzer.TokenStream("Denmark", new StringReader(sortData[i][5]))));
}
writer.AddDocument(doc);
}
writer.ForceMerge(1);
writer.Dispose();
IndexReader reader = DirectoryReader.Open(indexStore);
IndexSearcher searcher = new IndexSearcher(reader);
Sort sort = new Sort();
Query queryX = new TermQuery(new Term("contents", "x"));
Query queryY = new TermQuery(new Term("contents", "y"));
sort.SetSort(new SortField("US", SortField.Type_e.STRING));
AssertMatches(searcher, queryY, sort, usResult);
sort.SetSort(new SortField("France", SortField.Type_e.STRING));
AssertMatches(searcher, queryX, sort, frResult);
sort.SetSort(new SortField("Sweden", SortField.Type_e.STRING));
AssertMatches(searcher, queryY, sort, svResult);
sort.SetSort(new SortField("Denmark", SortField.Type_e.STRING));
AssertMatches(searcher, queryY, sort, dkResult);
reader.Dispose();
indexStore.Dispose();
}
public virtual void TestRAMDirectorySize()
{
Directory dir = NewFSDirectory(IndexDir);
MockDirectoryWrapper ramDir = new MockDirectoryWrapper(Random(), new RAMDirectory(dir, NewIOContext(Random())));
dir.Dispose();
IndexWriter writer = new IndexWriter(ramDir, (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetOpenMode(IndexWriterConfig.OpenMode_e.APPEND));
writer.ForceMerge(1);
Assert.AreEqual(ramDir.SizeInBytes(), ramDir.RecomputedSizeInBytes);
ThreadClass[] threads = new ThreadClass[NumThreads];
for (int i = 0; i < NumThreads; i++)
{
int num = i;
threads[i] = new ThreadAnonymousInnerClassHelper(this, writer, num);
}
for (int i = 0; i < NumThreads; i++)
{
threads[i].Start();
}
for (int i = 0; i < NumThreads; i++)
{
threads[i].Join();
}
writer.ForceMerge(1);
Assert.AreEqual(ramDir.SizeInBytes(), ramDir.RecomputedSizeInBytes);
writer.Dispose();
}
public virtual void CreateRandomTerms(int nDocs, int nTerms, double power, Directory dir)
{
int[] freq = new int[nTerms];
Terms = new Term[nTerms];
for (int i = 0; i < nTerms; i++)
{
int f = (nTerms + 1) - i; // make first terms less frequent
freq[i] = (int)Math.Ceiling(Math.Pow(f, power));
Terms[i] = new Term("f", char.ToString((char)('A' + i)));
}
IndexWriter iw = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.CREATE));
for (int i = 0; i < nDocs; i++)
{
Document d = new Document();
for (int j = 0; j < nTerms; j++)
{
if (Random().Next(freq[j]) == 0)
{
d.Add(NewStringField("f", Terms[j].Text(), Field.Store.NO));
//System.out.println(d);
}
}
iw.AddDocument(d);
}
iw.ForceMerge(1);
iw.Dispose();
}
public virtual void TestPerFieldCodec()
{
int NUM_DOCS = AtLeast(173);
if (VERBOSE)
{
Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS);
}
using (BaseDirectoryWrapper dir = NewDirectory())
{
dir.CheckIndexOnClose = false; // we use a custom codec provider
using (IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(new CustomPerFieldCodec()).SetMergePolicy(NewLogMergePolicy(3))))
{
Documents.Document doc = new Documents.Document();
// uses default codec:
doc.Add(NewTextField("field1", "this field uses the standard codec as the test", Field.Store.NO));
// uses pulsing codec:
Field field2 = NewTextField("field2", "this field uses the pulsing codec as the test", Field.Store.NO);
doc.Add(field2);
Field idField = NewStringField("id", "", Field.Store.NO);
doc.Add(idField);
for (int i = 0; i < NUM_DOCS; i++)
{
idField.StringValue = "" + i;
w.AddDocument(doc);
if ((i + 1) % 10 == 0)
{
w.Commit();
}
}
if (VERBOSE)
{
Console.WriteLine("TEST: now delete id=77");
}
w.DeleteDocuments(new Term("id", "77"));
using (IndexReader r = DirectoryReader.Open(w, true))
{
Assert.AreEqual(NUM_DOCS - 1, r.NumDocs);
IndexSearcher s = NewSearcher(r);
Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits);
Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits);
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: now delete 2nd doc");
}
w.DeleteDocuments(new Term("id", "44"));
if (VERBOSE)
{
Console.WriteLine("\nTEST: now force merge");
}
w.ForceMerge(1);
if (VERBOSE)
{
Console.WriteLine("\nTEST: now open reader");
}
using (IndexReader r = DirectoryReader.Open(w, true))
{
Assert.AreEqual(NUM_DOCS - 2, r.MaxDoc);
Assert.AreEqual(NUM_DOCS - 2, r.NumDocs);
IndexSearcher s = NewSearcher(r);
Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits);
Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits);
Assert.AreEqual(1, s.Search(new TermQuery(new Term("id", "76")), 1).TotalHits);
Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "77")), 1).TotalHits);
Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "44")), 1).TotalHits);
if (VERBOSE)
{
Console.WriteLine("\nTEST: now close NRT reader");
}
}
}
}
}
