public override void SetUp()
{
base.SetUp();
dir = NewDirectory();
iw = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
Document doc = new Document();
Field idField = new StringField("id", "", Field.Store.NO);
doc.Add(idField);
// add 500 docs with id 0..499
for (int i = 0; i < 500; i++)
{
idField.SetStringValue(Convert.ToString(i));
iw.AddDocument(doc);
}
// delete 20 of them
for (int i = 0; i < 20; i++)
{
iw.DeleteDocuments(new Term("id", Convert.ToString(Random.Next(iw.MaxDoc))));
}
ir = iw.GetReader();
@is = NewSearcher(ir);
}
public override void SetUp()
{
base.SetUp();
dir = NewDirectory();
IndexWriterConfig iwConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwConfig.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random, dir, iwConfig);
Document document = new Document();
Field idField = new StringField("id", "", Field.Store.NO);
document.Add(idField);
Field byteField = new StringField("byte", "", Field.Store.NO);
document.Add(byteField);
Field doubleField = new StringField("double", "", Field.Store.NO);
document.Add(doubleField);
Field floatField = new StringField("float", "", Field.Store.NO);
document.Add(floatField);
Field intField = new StringField("int", "", Field.Store.NO);
document.Add(intField);
Field longField = new StringField("long", "", Field.Store.NO);
document.Add(longField);
Field shortField = new StringField("short", "", Field.Store.NO);
document.Add(shortField);
Field stringField = new StringField("string", "", Field.Store.NO);
document.Add(stringField);
Field textField = new TextField("text", "", Field.Store.NO);
document.Add(textField);
foreach (string[] doc in documents)
{
idField.SetStringValue(doc[0]);
byteField.SetStringValue(doc[1]);
doubleField.SetStringValue(doc[2]);
floatField.SetStringValue(doc[3]);
intField.SetStringValue(doc[4]);
longField.SetStringValue(doc[5]);
shortField.SetStringValue(doc[6]);
stringField.SetStringValue(doc[7]);
textField.SetStringValue(doc[8]);
iw.AddDocument(document);
}
reader = iw.GetReader();
searcher = NewSearcher(reader);
iw.Dispose();
}
public override void BeforeClass()
{
base.BeforeClass();
Random random = Random;
m_directory = NewDirectory();
m_stopword = "" + GetRandomChar();
CharacterRunAutomaton stopset = new CharacterRunAutomaton(BasicAutomata.MakeString(m_stopword));
m_analyzer = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false, stopset);
RandomIndexWriter iw = new RandomIndexWriter(random, m_directory, m_analyzer);
Document doc = new Document();
Field id = new StringField("id", "", Field.Store.NO);
Field field = new TextField("field", "", Field.Store.NO);
doc.Add(id);
doc.Add(field);
// index some docs
int numDocs = AtLeast(1000);
for (int i = 0; i < numDocs; i++)
{
id.SetStringValue(Convert.ToString(i, CultureInfo.InvariantCulture));
field.SetStringValue(RandomFieldContents());
iw.AddDocument(doc);
}
// delete some docs
int numDeletes = numDocs / 20;
for (int i = 0; i < numDeletes; i++)
{
Term toDelete = new Term("id", Convert.ToString(random.Next(numDocs), CultureInfo.InvariantCulture));
if (random.NextBoolean())
{
iw.DeleteDocuments(toDelete);
}
else
{
iw.DeleteDocuments(new TermQuery(toDelete));
}
}
m_reader = iw.GetReader();
m_s1 = NewSearcher(m_reader);
m_s2 = NewSearcher(m_reader);
iw.Dispose();
}
public void TestRanking()
{
// number of documents: we will check each one
int numDocs = AtLeast(100);
// number of top-N snippets, we will check 1 .. N
int maxTopN = 5;
// maximum number of elements to put in a sentence.
int maxSentenceLength = 10;
// maximum number of sentences in a document
int maxNumSentences = 20;
Directory dir = NewDirectory();
RandomIndexWriter iw = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir, new MockAnalyzer(Random, MockTokenizer.SIMPLE, true));
Document document = new Document();
Field id = new StringField("id", "", Field.Store.NO);
FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
offsetsType.IndexOptions = (IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
Field body = new Field("body", "", offsetsType);
document.Add(id);
document.Add(body);
for (int i = 0; i < numDocs; i++)
{
StringBuilder bodyText = new StringBuilder();
int numSentences = TestUtil.NextInt32(Random, 1, maxNumSentences);
for (int j = 0; j < numSentences; j++)
{
bodyText.Append(newSentence(Random, maxSentenceLength));
}
body.SetStringValue(bodyText.ToString());
id.SetStringValue(i.ToString(CultureInfo.InvariantCulture));
iw.AddDocument(document);
}
IndexReader ir = iw.GetReader();
IndexSearcher searcher = NewSearcher(ir);
for (int i = 0; i < numDocs; i++)
{
checkDocument(searcher, i, maxTopN);
}
iw.Dispose();
ir.Dispose();
dir.Dispose();
}
// [Test, Timeout(300000)] // LUCENENET NOTE: For now, we are overriding this test in every subclass to pull it into the right context for the subclass
public virtual void TestConcurrentReads()
{
Directory dir = NewDirectory();
IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwConf.SetMaxBufferedDocs(RandomInts.NextIntBetween(Random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConf);
// make sure the readers are properly cloned
Document doc = new Document();
Field field = new StringField("fld", "", Field.Store.YES);
doc.Add(field);
int numDocs = AtLeast(1000);
for (int i = 0; i < numDocs; ++i)
{
field.SetStringValue("" + i);
iw.AddDocument(doc);
}
iw.Commit();
DirectoryReader rd = DirectoryReader.Open(dir);
IndexSearcher searcher = new IndexSearcher(rd);
int concurrentReads = AtLeast(5);
int readsPerThread = AtLeast(50);
IList <ThreadClass> readThreads = new List <ThreadClass>();
AtomicObject <Exception> ex = new AtomicObject <Exception>();
for (int i = 0; i < concurrentReads; ++i)
{
readThreads.Add(new ThreadAnonymousInnerClassHelper(numDocs, rd, searcher, readsPerThread, ex, i));
}
foreach (ThreadClass thread in readThreads)
{
thread.Start();
}
foreach (ThreadClass thread in readThreads)
{
thread.Join();
}
rd.Dispose();
if (ex.Value != null)
{
throw ex.Value;
}
iw.Dispose();
dir.Dispose();
}
public virtual void TestConcurrentReads()
{
using Directory dir = NewDirectory();
IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwConf.SetMaxBufferedDocs(RandomInts.RandomInt32Between(Random, 2, 30));
using RandomIndexWriter iw = new RandomIndexWriter(Random, dir, iwConf);
// make sure the readers are properly cloned
Document doc = new Document();
Field field = new StringField("fld", "", Field.Store.YES);
doc.Add(field);
int numDocs = AtLeast(1000);
for (int i = 0; i < numDocs; ++i)
{
field.SetStringValue("" + i);
iw.AddDocument(doc);
}
iw.Commit();
AtomicReference <Exception> ex = new AtomicReference <Exception>();
using (DirectoryReader rd = DirectoryReader.Open(dir))
{
IndexSearcher searcher = new IndexSearcher(rd);
int concurrentReads = AtLeast(5);
int readsPerThread = AtLeast(50);
IList <ThreadJob> readThreads = new JCG.List <ThreadJob>();
for (int i = 0; i < concurrentReads; ++i)
{
readThreads.Add(new ThreadAnonymousClass(numDocs, rd, searcher, readsPerThread, ex));
}
foreach (ThreadJob thread in readThreads)
{
thread.Start();
}
foreach (ThreadJob thread in readThreads)
{
thread.Join();
}
} // rd.Dispose();
if (ex.Value != null)
{
ExceptionDispatchInfo.Capture(ex.Value).Throw(); // LUCENENET: Rethrow to preserve stack details from the other thread
}
}
public override void SetUp()
{
base.SetUp();
Dir = NewDirectory();
Iw = new RandomIndexWriter(Random(), Dir, Similarity, TimeZone);
Document doc = new Document();
Field idField = new StringField("id", "", Field.Store.NO);
doc.Add(idField);
// add 500 docs with id 0..499
for (int i = 0; i < 500; i++)
{
idField.SetStringValue(Convert.ToString(i));
Iw.AddDocument(doc);
}
// delete 20 of them
for (int i = 0; i < 20; i++)
{
Iw.DeleteDocuments(new Term("id", Convert.ToString(Random().Next(Iw.MaxDoc))));
}
Ir = Iw.Reader;
@is = NewSearcher(Ir);
}
public void TestSearchAfterWhenSortingByFunctionValues()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy()); // depends on docid order
RandomIndexWriter writer = new RandomIndexWriter(Random, dir, iwc);
Document doc = new Document();
Field field = new StringField("value", "", Field.Store.YES);
doc.Add(field);
// Save docs unsorted (decreasing value n, n-1, ...)
const int NUM_VALS = 5;
for (int val = NUM_VALS; val > 0; val--)
{
field.SetStringValue(Convert.ToString(val));
writer.AddDocument(doc);
}
// Open index
IndexReader reader = writer.GetReader();
writer.Dispose();
IndexSearcher searcher = NewSearcher(reader);
// Get ValueSource from FieldCache
Int32FieldSource src = new Int32FieldSource("value");
// ...and make it a sort criterion
SortField sf = src.GetSortField(false).Rewrite(searcher);
Sort orderBy = new Sort(sf);
// Get hits sorted by our FunctionValues (ascending values)
Query q = new MatchAllDocsQuery();
TopDocs hits = searcher.Search(q, reader.MaxDoc, orderBy);
assertEquals(NUM_VALS, hits.ScoreDocs.Length);
// Verify that sorting works in general
int i = 0;
foreach (ScoreDoc hit in hits.ScoreDocs)
{
int valueFromDoc = Convert.ToInt32(reader.Document(hit.Doc).Get("value"));
assertEquals(++i, valueFromDoc);
}
// Now get hits after hit #2 using IS.searchAfter()
int afterIdx = 1;
FieldDoc afterHit = (FieldDoc)hits.ScoreDocs[afterIdx];
hits = searcher.SearchAfter(afterHit, q, reader.MaxDoc, orderBy);
// Expected # of hits: NUM_VALS - 2
assertEquals(NUM_VALS - (afterIdx + 1), hits.ScoreDocs.Length);
// Verify that hits are actually "after"
int afterValue = (int)((double?)afterHit.Fields[0]);
foreach (ScoreDoc hit in hits.ScoreDocs)
{
int val = Convert.ToInt32(reader.Document(hit.Doc).Get("value"));
assertTrue(afterValue <= val);
assertFalse(hit.Doc == afterHit.Doc);
}
reader.Dispose();
dir.Dispose();
}
// [Test, LongRunningTest, Timeout(int.MaxValue)] // LUCENENET NOTE: For now, we are overriding this test in every subclass to pull it into the right context for the subclass
public virtual void TestBigDocuments()
{
// "big" as "much bigger than the chunk size"
// for this test we force a FS dir
// we can't just use newFSDirectory, because this test doesn't really index anything.
// so if we get NRTCachingDir+SimpleText, we make massive stored fields and OOM (LUCENE-4484)
Directory dir = new MockDirectoryWrapper(Random(), new MMapDirectory(CreateTempDir("testBigDocuments")));
IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwConf.SetMaxBufferedDocs(RandomInts.NextIntBetween(Random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConf);
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
Document emptyDoc = new Document(); // emptyDoc
Document bigDoc1 = new Document(); // lot of small fields
Document bigDoc2 = new Document(); // 1 very big field
Field idField = new StringField("id", "", Field.Store.NO);
emptyDoc.Add(idField);
bigDoc1.Add(idField);
bigDoc2.Add(idField);
FieldType onlyStored = new FieldType(StringField.TYPE_STORED);
onlyStored.IsIndexed = false;
Field smallField = new Field("fld", RandomByteArray(Random().Next(10), 256), onlyStored);
int numFields = RandomInts.NextIntBetween(Random(), 500000, 1000000);
for (int i = 0; i < numFields; ++i)
{
bigDoc1.Add(smallField);
}
Field bigField = new Field("fld", RandomByteArray(RandomInts.NextIntBetween(Random(), 1000000, 5000000), 2), onlyStored);
bigDoc2.Add(bigField);
int numDocs = AtLeast(5);
Document[] docs = new Document[numDocs];
for (int i = 0; i < numDocs; ++i)
{
docs[i] = RandomInts.RandomFrom(Random(), Arrays.AsList(emptyDoc, bigDoc1, bigDoc2));
}
for (int i = 0; i < numDocs; ++i)
{
idField.SetStringValue("" + i);
iw.AddDocument(docs[i]);
if (Random().Next(numDocs) == 0)
{
iw.Commit();
}
}
iw.Commit();
iw.ForceMerge(1); // look at what happens when big docs are merged
DirectoryReader rd = DirectoryReader.Open(dir);
IndexSearcher searcher = new IndexSearcher(rd);
for (int i = 0; i < numDocs; ++i)
{
Query query = new TermQuery(new Term("id", "" + i));
TopDocs topDocs = searcher.Search(query, 1);
Assert.AreEqual(1, topDocs.TotalHits, "" + i);
Document doc = rd.Document(topDocs.ScoreDocs[0].Doc);
Assert.IsNotNull(doc);
IIndexableField[] fieldValues = doc.GetFields("fld");
Assert.AreEqual(docs[i].GetFields("fld").Length, fieldValues.Length);
if (fieldValues.Length > 0)
{
Assert.AreEqual(docs[i].GetFields("fld")[0].GetBinaryValue(), fieldValues[0].GetBinaryValue());
}
}
rd.Dispose();
iw.Dispose();
dir.Dispose();
}
