private void CreateRandomIndexes(int maxSegments)
{
dir = NewDirectory();
numDocs = AtLeast(150);
int numTerms = TestUtil.NextInt(Random(), 1, numDocs / 5);
ISet<string> randomTerms = new HashSet<string>();
while (randomTerms.size() < numTerms)
{
randomTerms.add(TestUtil.RandomSimpleString(Random()));
}
terms = new List<string>(randomTerms);
int seed = Random().Next();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(new Random(seed)));
iwc.SetMergePolicy(TestSortingMergePolicy.NewSortingMergePolicy(sort));
iw = new RandomIndexWriter(new Random(seed), dir, iwc);
for (int i = 0; i < numDocs; ++i)
{
Document doc = RandomDocument();
iw.AddDocument(doc);
if (i == numDocs / 2 || (i != numDocs - 1 && Random().nextInt(8) == 0))
{
iw.Commit();
}
if (Random().nextInt(15) == 0)
{
string term = RandomInts.RandomFrom(Random(), terms);
iw.DeleteDocuments(new Term("s", term));
}
}
reader = iw.Reader;
}
public override void SetUp()
{
base.SetUp();
dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, true), Similarity, TimeZone);
for (int i = 900; i < 1112; i++)
{
Document doc = new Document();
string num = Regex.Replace(Regex.Replace(English.IntToEnglish(i), "[-]", " "), "[,]", "");
doc.Add(NewTextField("numbers", num, Field.Store.NO));
writer.AddDocument(doc);
}
{
Document doc = new Document();
doc.Add(NewTextField("numbers", "thou hast sand betwixt thy toes", Field.Store.NO));
writer.AddDocument(doc);
}
{
Document doc = new Document();
doc.Add(NewTextField("numbers", "hundredeight eightyeight yeight", Field.Store.NO));
writer.AddDocument(doc);
}
{
Document doc = new Document();
doc.Add(NewTextField("numbers", "tres y cinco", Field.Store.NO));
writer.AddDocument(doc);
}
writer.Commit();
writer.Dispose();
}
public virtual void TestReuseDocsEnumNoReuse()
{
Directory dir = NewDirectory();
Codec cp = TestUtil.AlwaysPostingsFormat(new Lucene40RWPostingsFormat());
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(cp));
int numdocs = AtLeast(20);
CreateRandomIndex(numdocs, writer, Random());
writer.Commit();
DirectoryReader open = DirectoryReader.Open(dir);
foreach (AtomicReaderContext ctx in open.Leaves())
{
AtomicReader indexReader = (AtomicReader)ctx.Reader();
Terms terms = indexReader.Terms("body");
TermsEnum iterator = terms.Iterator(null);
IdentityHashMap<DocsEnum, bool?> enums = new IdentityHashMap<DocsEnum, bool?>();
MatchNoBits bits = new MatchNoBits(indexReader.MaxDoc());
while ((iterator.Next()) != null)
{
DocsEnum docs = iterator.Docs(Random().NextBoolean() ? bits : new MatchNoBits(indexReader.MaxDoc()), null, Random().NextBoolean() ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE);
enums[docs] = true;
}
Assert.AreEqual(terms.Size(), enums.Count);
}
IOUtils.Close(writer, open, dir);
}
public override void SetUp()
{
base.SetUp();
_dir = NewDirectory();
_indexWriter = new RandomIndexWriter(Random(), _dir, new MockAnalyzer(Random()), Similarity, TimeZone);
FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.StoreTermVectors = true;
ft.StoreTermVectorOffsets = true;
ft.StoreTermVectorPositions = true;
Analyzer analyzer = new MockAnalyzer(Random());
Document doc;
for (int i = 0; i < 100; i++)
{
doc = new Document();
doc.Add(new Field(_idFieldName, Random().toString(), ft));
doc.Add(new Field(_textFieldName, new StringBuilder(Random().toString()).append(Random().toString()).append(
Random().toString()).toString(), ft));
doc.Add(new Field(_classFieldName, Random().toString(), ft));
_indexWriter.AddDocument(doc, analyzer);
}
_indexWriter.Commit();
_originalIndex = SlowCompositeReaderWrapper.Wrap(_indexWriter.Reader);
}
public virtual void Test()
{
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
conf.SetCodec(new Lucene46Codec());
RandomIndexWriter riw = new RandomIndexWriter(Random(), dir, conf);
Document doc = new Document();
// these fields should sometimes get term vectors, etc
Field idField = NewStringField("id", "", Field.Store.NO);
Field bodyField = NewTextField("body", "", Field.Store.NO);
Field dvField = new NumericDocValuesField("dv", 5);
doc.Add(idField);
doc.Add(bodyField);
doc.Add(dvField);
for (int i = 0; i < 100; i++)
{
idField.StringValue = Convert.ToString(i);
bodyField.StringValue = TestUtil.RandomUnicodeString(Random());
riw.AddDocument(doc);
if (Random().Next(7) == 0)
{
riw.Commit();
}
// TODO: we should make a new format with a clean header...
// if (Random().nextInt(20) == 0) {
// riw.DeleteDocuments(new Term("id", Integer.toString(i)));
// }
}
riw.Dispose();
CheckHeaders(dir);
dir.Dispose();
}
public virtual void TestIndexing()
{
DirectoryInfo tmpDir = CreateTempDir("TestNeverDelete");
BaseDirectoryWrapper d = NewFSDirectory(tmpDir);
// We want to "see" files removed if Lucene removed
// them. this is still worth running on Windows since
// some files the IR opens and closes.
if (d is MockDirectoryWrapper)
{
((MockDirectoryWrapper)d).NoDeleteOpenFile = false;
}
RandomIndexWriter w = new RandomIndexWriter(Random(), d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
w.w.Config.SetMaxBufferedDocs(TestUtil.NextInt(Random(), 5, 30));
w.Commit();
ThreadClass[] indexThreads = new ThreadClass[Random().Next(4)];
long stopTime = Environment.TickCount + AtLeast(1000);
for (int x = 0; x < indexThreads.Length; x++)
{
indexThreads[x] = new ThreadAnonymousInnerClassHelper(w, stopTime);
indexThreads[x].Name = "Thread " + x;
indexThreads[x].Start();
}
HashSet<string> allFiles = new HashSet<string>();
DirectoryReader r = DirectoryReader.Open(d);
while (Environment.TickCount < stopTime)
{
IndexCommit ic = r.IndexCommit;
if (VERBOSE)
{
Console.WriteLine("TEST: check files: " + ic.FileNames);
}
allFiles.AddAll(ic.FileNames);
// Make sure no old files were removed
foreach (string fileName in allFiles)
{
Assert.IsTrue(SlowFileExists(d, fileName), "file " + fileName + " does not exist");
}
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
if (r2 != null)
{
r.Dispose();
r = r2;
}
Thread.Sleep(1);
}
r.Dispose();
foreach (ThreadClass t in indexThreads)
{
t.Join();
}
w.Dispose();
d.Dispose();
System.IO.Directory.Delete(tmpDir.FullName, true);
}
public virtual void TestSomeSegmentsMissing()
{
AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet);
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
FacetsConfig config = new FacetsConfig();
Document doc = new Document();
doc.Add(new SortedSetDocValuesFacetField("a", "foo1"));
writer.AddDocument(config.Build(doc));
writer.Commit();
doc = new Document();
writer.AddDocument(config.Build(doc));
writer.Commit();
doc = new Document();
doc.Add(new SortedSetDocValuesFacetField("a", "foo2"));
writer.AddDocument(config.Build(doc));
writer.Commit();
// NRT open
IndexSearcher searcher = NewSearcher(writer.GetReader());
writer.Dispose();
// Per-top-reader state:
SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);
FacetsCollector c = new FacetsCollector();
searcher.Search(new MatchAllDocsQuery(), c);
SortedSetDocValuesFacetCounts facets = new SortedSetDocValuesFacetCounts(state, c);
// Ask for top 10 labels for any dims that have counts:
Assert.AreEqual("dim=a path=[] value=2 childCount=2\n foo1 (1)\n foo2 (1)\n", facets.GetTopChildren(10, "a").ToString());
searcher.IndexReader.Dispose();
dir.Dispose();
}
public virtual void TestDeletePartiallyWrittenFilesIfAbort()
{
Directory dir = NewDirectory();
IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwConf.SetMaxBufferedDocs(RandomInts.NextIntBetween(Random(), 2, 30));
iwConf.SetCodec(CompressingCodec.RandomInstance(Random()));
// disable CFS because this test checks file names
iwConf.SetMergePolicy(NewLogMergePolicy(false));
iwConf.SetUseCompoundFile(false);
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConf);
Document validDoc = new Document();
validDoc.Add(new IntField("id", 0, Field.Store.YES));
iw.AddDocument(validDoc);
iw.Commit();
// make sure that #writeField will fail to trigger an abort
Document invalidDoc = new Document();
FieldType fieldType = new FieldType();
fieldType.Stored = true;
invalidDoc.Add(new FieldAnonymousInnerClassHelper(this, fieldType));
try
{
iw.AddDocument(invalidDoc);
iw.Commit();
}
finally
{
int counter = 0;
foreach (string fileName in dir.ListAll())
{
if (fileName.EndsWith(".fdt") || fileName.EndsWith(".fdx"))
{
counter++;
}
}
// Only one .fdt and one .fdx files must have been found
Assert.AreEqual(2, counter);
iw.Dispose();
dir.Dispose();
}
}
public virtual void TestBasic()
{
AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet);
Directory dir = NewDirectory();
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("a", true);
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
Document doc = new Document();
doc.Add(new SortedSetDocValuesFacetField("a", "foo"));
doc.Add(new SortedSetDocValuesFacetField("a", "bar"));
doc.Add(new SortedSetDocValuesFacetField("a", "zoo"));
doc.Add(new SortedSetDocValuesFacetField("b", "baz"));
writer.AddDocument(config.Build(doc));
if (Random.NextBoolean())
{
writer.Commit();
}
doc = new Document();
doc.Add(new SortedSetDocValuesFacetField("a", "foo"));
writer.AddDocument(config.Build(doc));
// NRT open
IndexSearcher searcher = NewSearcher(writer.GetReader());
// Per-top-reader state:
SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);
FacetsCollector c = new FacetsCollector();
searcher.Search(new MatchAllDocsQuery(), c);
SortedSetDocValuesFacetCounts facets = new SortedSetDocValuesFacetCounts(state, c);
Assert.AreEqual("dim=a path=[] value=4 childCount=3\n foo (2)\n bar (1)\n zoo (1)\n", facets.GetTopChildren(10, "a").ToString());
Assert.AreEqual("dim=b path=[] value=1 childCount=1\n baz (1)\n", facets.GetTopChildren(10, "b").ToString());
// DrillDown:
DrillDownQuery q = new DrillDownQuery(config);
q.Add("a", "foo");
q.Add("b", "baz");
TopDocs hits = searcher.Search(q, 1);
Assert.AreEqual(1, hits.TotalHits);
IOUtils.Dispose(writer, searcher.IndexReader, dir);
}
public virtual void TestFloatNorms()
{
Directory dir = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig config = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
Similarity provider = new MySimProvider(this);
config.SetSimilarity(provider);
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, config);
LineFileDocs docs = new LineFileDocs(Random());
int num = AtLeast(100);
for (int i = 0; i < num; i++)
{
Document doc = docs.NextDoc();
float nextFloat = (float)Random().NextDouble();
// Cast to a double to get more precision output to the string.
Field f = new TextField(FloatTestField, "" + (double)nextFloat, Field.Store.YES);
f.Boost = nextFloat;
doc.Add(f);
writer.AddDocument(doc);
doc.RemoveField(FloatTestField);
if (Rarely())
{
writer.Commit();
}
}
writer.Commit();
writer.Dispose();
AtomicReader open = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir));
NumericDocValues norms = open.GetNormValues(FloatTestField);
Assert.IsNotNull(norms);
for (int i = 0; i < open.MaxDoc; i++)
{
Document document = open.Document(i);
float expected = Convert.ToSingle(document.Get(FloatTestField));
Assert.AreEqual(expected, Number.IntBitsToFloat((int)norms.Get(i)), 0.0f);
}
open.Dispose();
dir.Dispose();
docs.Dispose();
}
public void TestWithContexts()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
KeyValuePair <List <string>, IDictionary <string, Document> > res = GenerateIndexDocuments(AtLeast(1000), true, true);
IDictionary <string, Document> docs = res.Value;
List <string> invalidDocTerms = res.Key;
foreach (Document doc in docs.Values)
{
writer.AddDocument(doc);
}
writer.Commit();
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
IDictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME, CONTEXT_FIELD_NAME);
IInputIterator inputIterator = dictionary.GetEntryIterator();
BytesRef f;
while ((f = inputIterator.Next()) != null)
{
string field = f.Utf8ToString();
Document doc = docs.ContainsKey(field) ? docs[field] : null;
docs.Remove(field);
//Document doc = docs.remove(f.utf8ToString());
assertTrue(f.equals(new BytesRef(doc.Get(FIELD_NAME))));
IIndexableField weightField = doc.GetField(WEIGHT_FIELD_NAME);
assertEquals(inputIterator.Weight, (weightField != null) ? Convert.ToInt64(weightField.GetNumericValue()) : 0);
assertTrue(inputIterator.Payload.equals(doc.GetField(PAYLOAD_FIELD_NAME).GetBinaryValue()));
ISet <BytesRef> oriCtxs = new HashSet <BytesRef>();
IEnumerable <BytesRef> contextSet = inputIterator.Contexts;
foreach (IIndexableField ctxf in doc.GetFields(CONTEXT_FIELD_NAME))
{
oriCtxs.add(ctxf.GetBinaryValue());
}
assertEquals(oriCtxs.size(), contextSet.Count());
}
foreach (string invalidTerm in invalidDocTerms)
{
var invalid = docs[invalidTerm];
docs.Remove(invalidTerm);
assertNotNull(invalid);
}
assertTrue(!docs.Any());
ir.Dispose();
dir.Dispose();
}
public void TestWithContext()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter writer = new RandomIndexWriter(Random, dir, iwc);
IDictionary <string, Document> docs = GenerateIndexDocuments(AtLeast(100));
foreach (Document doc in docs.Values)
{
writer.AddDocument(doc);
}
writer.Commit();
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
ValueSource[] toAdd = new ValueSource[] { new Int64FieldSource(WEIGHT_FIELD_NAME_1), new Int64FieldSource(WEIGHT_FIELD_NAME_2), new Int64FieldSource(WEIGHT_FIELD_NAME_3) };
IDictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumSingleFunction(toAdd), PAYLOAD_FIELD_NAME, CONTEXTS_FIELD_NAME);
IInputEnumerator inputIterator = dictionary.GetEntryEnumerator();
while (inputIterator.MoveNext())
{
string field = inputIterator.Current.Utf8ToString();
Document doc = docs[field];
docs.Remove(field);
long w1 = doc.GetField(WEIGHT_FIELD_NAME_1).GetInt64ValueOrDefault();
long w2 = doc.GetField(WEIGHT_FIELD_NAME_2).GetInt64ValueOrDefault();
long w3 = doc.GetField(WEIGHT_FIELD_NAME_3).GetInt64ValueOrDefault();
assertTrue(inputIterator.Current.equals(new BytesRef(doc.Get(FIELD_NAME))));
assertEquals(inputIterator.Weight, (w1 + w2 + w3));
assertTrue(inputIterator.Payload.equals(doc.GetField(PAYLOAD_FIELD_NAME).GetBinaryValue()));
// LUCENENET NOTE: This test was once failing because we used SCG.HashSet<T> whose
// Equals() implementation does not check for set equality. As a result SortedInputEnumerator
// had been modified to reverse the results to get the test to pass. However, using JCG.HashSet<T>
// ensures that set equality (that is equality that doesn't care about order of items) is respected.
// SortedInputEnumerator has also had the specific sorting removed.
ISet <BytesRef> originalCtxs = new JCG.HashSet <BytesRef>();
foreach (IIndexableField ctxf in doc.GetFields(CONTEXTS_FIELD_NAME))
{
originalCtxs.add(ctxf.GetBinaryValue());
}
assertEquals(originalCtxs, inputIterator.Contexts);
}
assertTrue(docs.Count == 0);
ir.Dispose();
dir.Dispose();
}
public virtual void TestNegativeScores()
{
// The Top*Collectors previously filtered out documents with <= scores. this
// behavior has changed. this test checks that if PositiveOnlyScoresFilter
// wraps one of these collectors, documents with <= 0 scores are indeed
// filtered.
int numPositiveScores = 0;
for (int i = 0; i < Scores.Length; i++)
{
if (Scores[i] > 0)
{
++numPositiveScores;
}
}
Directory directory = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, directory);
writer.Commit();
IndexReader ir = writer.GetReader();
writer.Dispose();
IndexSearcher searcher = NewSearcher(ir);
Weight fake = (new TermQuery(new Term("fake", "weight"))).CreateWeight(searcher);
Scorer s = new SimpleScorer(fake);
TopDocsCollector <ScoreDoc> tdc = TopScoreDocCollector.Create(Scores.Length, true);
ICollector c = new PositiveScoresOnlyCollector(tdc);
c.SetScorer(s);
while (s.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
c.Collect(0);
}
TopDocs td = tdc.GetTopDocs();
ScoreDoc[] sd = td.ScoreDocs;
Assert.AreEqual(numPositiveScores, td.TotalHits);
for (int i = 0; i < sd.Length; i++)
{
Assert.IsTrue(sd[i].Score > 0, "only positive scores should return: " + sd[i].Score);
}
ir.Dispose();
directory.Dispose();
}
public override void SetUp()
{
base.SetUp();
// initialize directory
Directory = NewDirectory();
Writer = new RandomIndexWriter(Random(), Directory);
// write document
Document doc = new Document();
doc.Add(NewTextField("UUID", "1", Field.Store.YES));
Writer.AddDocument(doc);
Writer.Commit();
}
public virtual void TestBasic()
{
AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet());
Directory dir = NewDirectory();
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("a", true);
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
Document doc = new Document();
doc.Add(new SortedSetDocValuesFacetField("a", "foo"));
doc.Add(new SortedSetDocValuesFacetField("a", "bar"));
doc.Add(new SortedSetDocValuesFacetField("a", "zoo"));
doc.Add(new SortedSetDocValuesFacetField("b", "baz"));
writer.AddDocument(config.Build(doc));
if (Random().NextBoolean())
{
writer.Commit();
}
doc = new Document();
doc.Add(new SortedSetDocValuesFacetField("a", "foo"));
writer.AddDocument(config.Build(doc));
// NRT open
IndexSearcher searcher = NewSearcher(writer.Reader);
// Per-top-reader state:
SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);
FacetsCollector c = new FacetsCollector();
searcher.Search(new MatchAllDocsQuery(), c);
SortedSetDocValuesFacetCounts facets = new SortedSetDocValuesFacetCounts(state, c);
Assert.AreEqual("dim=a path=[] value=4 childCount=3\n foo (2)\n bar (1)\n zoo (1)\n", facets.GetTopChildren(10, "a").ToString());
Assert.AreEqual("dim=b path=[] value=1 childCount=1\n baz (1)\n", facets.GetTopChildren(10, "b").ToString());
// DrillDown:
DrillDownQuery q = new DrillDownQuery(config);
q.Add("a", "foo");
q.Add("b", "baz");
TopDocs hits = searcher.Search(q, 1);
Assert.AreEqual(1, hits.TotalHits);
IOUtils.Close(writer, searcher.IndexReader, dir);
}
public virtual void TestReuseDocsEnumDifferentReader()
{
Directory dir = NewDirectory();
Codec cp = TestUtil.AlwaysPostingsFormat(new Lucene40RWPostingsFormat());
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter writer = new RandomIndexWriter(Random, dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetCodec(cp));
int numdocs = AtLeast(20);
CreateRandomIndex(numdocs, writer, Random);
writer.Commit();
DirectoryReader firstReader = DirectoryReader.Open(dir);
DirectoryReader secondReader = DirectoryReader.Open(dir);
IList <AtomicReaderContext> leaves = firstReader.Leaves;
IList <AtomicReaderContext> leaves2 = secondReader.Leaves;
foreach (AtomicReaderContext ctx in leaves)
{
Terms terms = ((AtomicReader)ctx.Reader).GetTerms("body");
TermsEnum iterator = terms.GetEnumerator();
IDictionary <DocsEnum, bool?> enums = new JCG.Dictionary <DocsEnum, bool?>(IdentityEqualityComparer <DocsEnum> .Default);
MatchNoBits bits = new MatchNoBits(firstReader.MaxDoc);
iterator = terms.GetEnumerator();
DocsEnum docs = null;
BytesRef term = null;
while (iterator.MoveNext())
{
term = iterator.Term;
docs = iterator.Docs(null, RandomDocsEnum("body", term, leaves2, bits), Random.NextBoolean() ? DocsFlags.FREQS : DocsFlags.NONE);
enums[docs] = true;
}
Assert.AreEqual(terms.Count, enums.Count);
iterator = terms.GetEnumerator();
enums.Clear();
docs = null;
while (iterator.MoveNext())
{
term = iterator.Term;
docs = iterator.Docs(bits, RandomDocsEnum("body", term, leaves2, bits), Random.NextBoolean() ? DocsFlags.FREQS : DocsFlags.NONE);
enums[docs] = true;
}
Assert.AreEqual(terms.Count, enums.Count);
}
IOUtils.Dispose(writer, firstReader, secondReader, dir);
}
public virtual void TestReuseDocsEnumSameBitsOrNull()
{
Directory dir = NewDirectory();
Codec cp = TestUtil.AlwaysPostingsFormat(new Lucene40RWPostingsFormat());
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(cp));
int numdocs = AtLeast(20);
CreateRandomIndex(numdocs, writer, Random());
writer.Commit();
DirectoryReader open = DirectoryReader.Open(dir);
foreach (AtomicReaderContext ctx in open.Leaves)
{
Terms terms = ((AtomicReader)ctx.Reader).GetTerms("body");
TermsEnum iterator = terms.GetIterator(null);
IdentityHashMap <DocsEnum, bool?> enums = new IdentityHashMap <DocsEnum, bool?>();
MatchNoBits bits = new MatchNoBits(open.MaxDoc);
DocsEnum docs = null;
while ((iterator.Next()) != null)
{
docs = iterator.Docs(bits, docs, Random().NextBoolean() ? DocsFlags.FREQS : DocsFlags.NONE);
enums[docs] = true;
}
Assert.AreEqual(1, enums.Count);
enums.Clear();
iterator = terms.GetIterator(null);
docs = null;
while ((iterator.Next()) != null)
{
docs = iterator.Docs(new MatchNoBits(open.MaxDoc), docs, Random().NextBoolean() ? DocsFlags.FREQS : DocsFlags.NONE);
enums[docs] = true;
}
Assert.AreEqual(terms.Count, enums.Count);
enums.Clear();
iterator = terms.GetIterator(null);
docs = null;
while ((iterator.Next()) != null)
{
docs = iterator.Docs(null, docs, Random().NextBoolean() ? DocsFlags.FREQS : DocsFlags.NONE);
enums[docs] = true;
}
Assert.AreEqual(1, enums.Count);
}
IOUtils.Close(writer, open, dir);
}
public override void SetUp()
{
base.SetUp();
dir = NewDirectory();
writer = new RandomIndexWriter(Random, dir);
int numDocs = AtLeast(100);
for (int i = 0; i < numDocs; i++)
{
writer.AddDocument(new Document());
if (Rarely())
{
writer.Commit();
}
}
}
public virtual void TestReuseDocsEnumDifferentReader()
{
Directory dir = NewDirectory();
Codec cp = TestUtil.AlwaysPostingsFormat(new Lucene40RWPostingsFormat(OLD_FORMAT_IMPERSONATION_IS_ACTIVE));
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetCodec(cp));
int numdocs = AtLeast(20);
CreateRandomIndex(numdocs, writer, Random());
writer.Commit();
DirectoryReader firstReader = DirectoryReader.Open(dir);
DirectoryReader secondReader = DirectoryReader.Open(dir);
IList <AtomicReaderContext> leaves = firstReader.Leaves;
IList <AtomicReaderContext> leaves2 = secondReader.Leaves;
foreach (AtomicReaderContext ctx in leaves)
{
Terms terms = ((AtomicReader)ctx.Reader).Terms("body");
TermsEnum iterator = terms.Iterator(null);
IdentityHashMap <DocsEnum, bool?> enums = new IdentityHashMap <DocsEnum, bool?>();
MatchNoBits bits = new MatchNoBits(firstReader.MaxDoc);
iterator = terms.Iterator(null);
DocsEnum docs = null;
BytesRef term = null;
while ((term = iterator.Next()) != null)
{
docs = iterator.Docs(null, RandomDocsEnum("body", term, leaves2, bits), Random().NextBoolean() ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE);
enums[docs] = true;
}
Assert.AreEqual(terms.Size(), enums.Count);
iterator = terms.Iterator(null);
enums.Clear();
docs = null;
while ((term = iterator.Next()) != null)
{
docs = iterator.Docs(bits, RandomDocsEnum("body", term, leaves2, bits), Random().NextBoolean() ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE);
enums[docs] = true;
}
Assert.AreEqual(terms.Size(), enums.Count);
}
IOUtils.Close(writer, firstReader, secondReader, dir);
}
public void TestWithoutPayload()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter writer = new RandomIndexWriter(Random, dir, iwc);
KeyValuePair <List <string>, IDictionary <string, Document> > res = GenerateIndexDocuments(AtLeast(1000), false, false);
IDictionary <string, Document> docs = res.Value;
List <string> invalidDocTerms = res.Key;
foreach (Document doc in docs.Values)
{
writer.AddDocument(doc);
}
writer.Commit();
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
IDictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
IInputIterator inputIterator = dictionary.GetEntryIterator();
BytesRef f;
while ((f = inputIterator.Next()) != null)
{
var field = f.Utf8ToString();
Document doc = docs[field];
docs.Remove(field);
assertTrue(f.equals(new BytesRef(doc.Get(FIELD_NAME))));
IIndexableField weightField = doc.GetField(WEIGHT_FIELD_NAME);
assertEquals(inputIterator.Weight, (weightField != null) ? weightField.GetInt64ValueOrDefault() : 0);
assertEquals(inputIterator.Payload, null);
}
foreach (string invalidTerm in invalidDocTerms)
{
var invalid = docs[invalidTerm];
docs.Remove(invalidTerm);
assertNotNull(invalid);
}
assertTrue(!docs.Any());
ir.Dispose();
dir.Dispose();
}
public void TestWithContext()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
IDictionary <string, Document> docs = GenerateIndexDocuments(AtLeast(100));
foreach (Document doc in docs.Values)
{
writer.AddDocument(doc);
}
writer.Commit();
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
ValueSource[] toAdd = new ValueSource[] { new Int64FieldSource(WEIGHT_FIELD_NAME_1), new Int64FieldSource(WEIGHT_FIELD_NAME_2), new Int64FieldSource(WEIGHT_FIELD_NAME_3) };
IDictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumSingleFunction(toAdd), PAYLOAD_FIELD_NAME, CONTEXTS_FIELD_NAME);
IInputIterator inputIterator = dictionary.GetEntryIterator();
BytesRef f;
while ((f = inputIterator.Next()) != null)
{
string field = f.Utf8ToString();
Document doc = docs.ContainsKey(field) ? docs[field] : null;
docs.Remove(field);
long w1 = doc.GetField(WEIGHT_FIELD_NAME_1).GetInt64ValueOrDefault();
long w2 = doc.GetField(WEIGHT_FIELD_NAME_2).GetInt64ValueOrDefault();
long w3 = doc.GetField(WEIGHT_FIELD_NAME_3).GetInt64ValueOrDefault();
assertTrue(f.equals(new BytesRef(doc.Get(FIELD_NAME))));
assertEquals(inputIterator.Weight, (w1 + w2 + w3));
assertTrue(inputIterator.Payload.equals(doc.GetField(PAYLOAD_FIELD_NAME).GetBinaryValue()));
ISet <BytesRef> originalCtxs = new HashSet <BytesRef>();
foreach (IIndexableField ctxf in doc.GetFields(CONTEXTS_FIELD_NAME))
{
originalCtxs.add(ctxf.GetBinaryValue());
}
assertEquals(originalCtxs, inputIterator.Contexts);
}
assertTrue(!docs.Any());
ir.Dispose();
dir.Dispose();
}
/** Creates an index for sorting. */
public void CreateIndex(Directory dir, int numDocs, Random random)
{
IList <int> ids = new List <int>();
for (int i = 0; i < numDocs; i++)
{
ids.Add(i * 10);
}
// shuffle them for indexing
// LUCENENET NOTE: Using LINQ, so we need to reassign the variable with the result
ids = CollectionsHelper.Shuffle(ids);
if (VERBOSE)
{
Console.WriteLine("Shuffled IDs for indexing: " + Arrays.ToString(ids.ToArray()));
}
PositionsTokenStream positions = new PositionsTokenStream();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
conf.SetMaxBufferedDocs(4); // create some segments
conf.SetSimilarity(new NormsSimilarity(conf.Similarity)); // for testing norms field
using (RandomIndexWriter writer = new RandomIndexWriter(random, dir, conf))
{
writer.RandomForceMerge = (false);
foreach (int id in ids)
{
writer.AddDocument(Doc(id, positions));
}
// delete some documents
writer.Commit();
foreach (int id in ids)
{
if (random.NextDouble() < 0.2)
{
if (VERBOSE)
{
Console.WriteLine("delete doc_id " + id);
}
writer.DeleteDocuments(new Term(ID_FIELD, id.ToString()));
}
}
}
}
public override void SetUp()
{
base.SetUp();
dir = NewDirectory();
writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
int numDocs = AtLeast(100);
for (int i = 0; i < numDocs; i++)
{
writer.AddDocument(new Document());
if (Rarely())
{
writer.Commit();
}
}
}
public virtual void TestBinary()
{
Directory dir = NewDirectory();
Document doc = new Document();
BytesRef @ref = new BytesRef();
Field field = new BinaryDocValuesField("bytes", @ref);
doc.Add(field);
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
@ref.CopyChars(TestUtil.RandomUnicodeString(Random()));
iw.AddDocument(doc);
if (Random().Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.Reader;
iw.ForceMerge(1);
DirectoryReader ir2 = iw.Reader;
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
BinaryDocValues multi = MultiDocValues.GetBinaryValues(ir, "bytes");
BinaryDocValues single = merged.GetBinaryDocValues("bytes");
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
for (int i = 0; i < numDocs; i++)
{
single.Get(i, expected);
multi.Get(i, actual);
Assert.AreEqual(expected, actual);
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}
public void TestBasic()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
KeyValuePair<List<string>, IDictionary<string, Document>> res = GenerateIndexDocuments(AtLeast(1000), true, false);
IDictionary<string, Document> docs = res.Value;
List<String> invalidDocTerms = res.Key;
foreach (Document doc in docs.Values)
{
writer.AddDocument(doc);
}
writer.Commit();
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
IDictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
IInputIterator inputIterator = dictionary.EntryIterator;
BytesRef f;
while ((f = inputIterator.Next()) != null)
{
string field = f.Utf8ToString();
Document doc = docs.ContainsKey(field) ? docs[field] : null;
docs.Remove(field);
//Document doc = docs.Remove(f.Utf8ToString());
assertTrue(f.equals(new BytesRef(doc.Get(FIELD_NAME))));
IndexableField weightField = doc.GetField(WEIGHT_FIELD_NAME);
assertEquals(inputIterator.Weight, (weightField != null) ? Convert.ToInt64(weightField.NumericValue) : 0);
assertTrue(inputIterator.Payload.Equals(doc.GetField(PAYLOAD_FIELD_NAME).BinaryValue));
}
foreach (string invalidTerm in invalidDocTerms)
{
var invalid = docs[invalidTerm];
docs.Remove(invalidTerm);
assertNotNull(invalid);
}
assertTrue(!docs.Any());
ir.Dispose();
dir.Dispose();
}
public void TestEmptyReader()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMergePolicy(NewLogMergePolicy());
// Make sure the index is created?
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
writer.Commit();
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
IDictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new DoubleConstValueSource(10), PAYLOAD_FIELD_NAME);
IInputIterator inputIterator = dictionary.EntryIterator;
assertNull(inputIterator.Next());
assertEquals(inputIterator.Weight, 0);
assertNull(inputIterator.Payload);
ir.Dispose();
dir.Dispose();
}
public void TestBasic()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
IDictionary <string, Document> docs = GenerateIndexDocuments(AtLeast(100));
foreach (Document doc in docs.Values)
{
writer.AddDocument(doc);
}
writer.Commit();
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
ValueSource[] toAdd = new ValueSource[] { new LongFieldSource(WEIGHT_FIELD_NAME_1), new LongFieldSource(WEIGHT_FIELD_NAME_2), new LongFieldSource(WEIGHT_FIELD_NAME_3) };
IDictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumFloatFunction(toAdd), PAYLOAD_FIELD_NAME);
IInputIterator inputIterator = dictionary.EntryIterator;
BytesRef f;
while ((f = inputIterator.Next()) != null)
{
string field = f.Utf8ToString();
Document doc = docs.ContainsKey(field) ? docs[field] : null;
docs.Remove(field);
//Document doc = docs.remove(f.utf8ToString());
long w1 = Convert.ToInt64(doc.GetField(WEIGHT_FIELD_NAME_1).NumericValue);
long w2 = Convert.ToInt64(doc.GetField(WEIGHT_FIELD_NAME_2).NumericValue);
long w3 = Convert.ToInt64(doc.GetField(WEIGHT_FIELD_NAME_3).NumericValue);
assertTrue(f.equals(new BytesRef(doc.Get(FIELD_NAME))));
assertEquals(inputIterator.Weight, (w1 + w2 + w3));
assertTrue(inputIterator.Payload.equals(doc.GetField(PAYLOAD_FIELD_NAME).BinaryValue));
}
assertTrue(!docs.Any());
ir.Dispose();
dir.Dispose();
}
public virtual void TestEvilSearcherFactory()
{
Random random = Random;
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
random, dir);
w.Commit();
IndexReader other = DirectoryReader.Open(dir);
SearcherFactory theEvilOne = new SearcherFactoryAnonymousInnerClassHelper3(this, other);
try
{
new SearcherManager(dir, theEvilOne);
}
#pragma warning disable 168
catch (InvalidOperationException ise)
#pragma warning restore 168
{
// expected
}
try
{
new SearcherManager(w.IndexWriter, random.NextBoolean(), theEvilOne);
}
#pragma warning disable 168
catch (InvalidOperationException ise)
#pragma warning restore 168
{
// expected
}
w.Dispose();
other.Dispose();
dir.Dispose();
}
public void TestWithoutPayload()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter writer = new RandomIndexWriter(Random, dir, iwc);
IDictionary <string, Document> docs = GenerateIndexDocuments(AtLeast(100));
foreach (Document doc in docs.Values)
{
writer.AddDocument(doc);
}
writer.Commit();
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
ValueSource[] toAdd = new ValueSource[] { new Int64FieldSource(WEIGHT_FIELD_NAME_1), new Int64FieldSource(WEIGHT_FIELD_NAME_2), new Int64FieldSource(WEIGHT_FIELD_NAME_3) };
IDictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumSingleFunction(toAdd));
IInputIterator inputIterator = dictionary.GetEntryIterator();
BytesRef f;
while ((f = inputIterator.Next()) != null)
{
string field = f.Utf8ToString();
Document doc = docs[field];
docs.Remove(field);
long w1 = doc.GetField(WEIGHT_FIELD_NAME_1).GetInt64ValueOrDefault();
long w2 = doc.GetField(WEIGHT_FIELD_NAME_2).GetInt64ValueOrDefault();
long w3 = doc.GetField(WEIGHT_FIELD_NAME_3).GetInt64ValueOrDefault();
assertTrue(f.equals(new BytesRef(doc.Get(FIELD_NAME))));
assertEquals(inputIterator.Weight, (w1 + w2 + w3));
assertEquals(inputIterator.Payload, null);
}
assertTrue(docs.Count == 0);
ir.Dispose();
dir.Dispose();
}
public void TestEmptyReader()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMergePolicy(NewLogMergePolicy());
// Make sure the index is created?
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
writer.Commit();
writer.Dispose();
IndexReader ir = DirectoryReader.Open(dir);
IDictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
IInputIterator inputIterator = dictionary.GetEntryIterator();
assertNull(inputIterator.Next());
assertEquals(inputIterator.Weight, 0);
assertNull(inputIterator.Payload);
ir.Dispose();
dir.Dispose();
}
public virtual void TestNumerics()
{
Directory dir = NewDirectory();
Document doc = new Document();
Field field = new NumericDocValuesField("numbers", 0);
doc.Add(field);
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
field.LongValue = Random().NextLong();
iw.AddDocument(doc);
if (Random().Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.Reader;
iw.ForceMerge(1);
DirectoryReader ir2 = iw.Reader;
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
NumericDocValues multi = MultiDocValues.GetNumericValues(ir, "numbers");
NumericDocValues single = merged.GetNumericDocValues("numbers");
for (int i = 0; i < numDocs; i++)
{
Assert.AreEqual(single.Get(i), multi.Get(i));
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}
public override void SetUp()
{
base.SetUp();
dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir, new MockAnalyzer(Random, MockTokenizer.WHITESPACE, true));
for (int i = 900; i < 1112; i++)
{
Document doc = new Document();
string num = Regex.Replace(Regex.Replace(English.Int32ToEnglish(i), "[-]", " "), "[,]", "");
doc.Add(NewTextField("numbers", num, Field.Store.NO));
writer.AddDocument(doc);
}
{
Document doc = new Document();
doc.Add(NewTextField("numbers", "thou hast sand betwixt thy toes", Field.Store.NO));
writer.AddDocument(doc);
}
{
Document doc = new Document();
doc.Add(NewTextField("numbers", "hundredeight eightyeight yeight", Field.Store.NO));
writer.AddDocument(doc);
}
{
Document doc = new Document();
doc.Add(NewTextField("numbers", "tres y cinco", Field.Store.NO));
writer.AddDocument(doc);
}
writer.Commit();
writer.Dispose();
}
public virtual void TestEvilSearcherFactory()
{
Random random = Random();
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(random, dir, Similarity, TimeZone);
w.Commit();
IndexReader other = DirectoryReader.Open(dir);
SearcherFactory theEvilOne = new SearcherFactoryAnonymousInnerClassHelper3(this, other);
try
{
new SearcherManager(dir, theEvilOne);
}
#pragma warning disable 168
catch (InvalidOperationException ise)
#pragma warning restore 168
{
// expected
}
try
{
new SearcherManager(w.w, random.NextBoolean(), theEvilOne);
}
#pragma warning disable 168
catch (InvalidOperationException ise)
#pragma warning restore 168
{
// expected
}
w.Dispose();
other.Dispose();
dir.Dispose();
}
public virtual void TestSlowCompositeReaderWrapper()
{
AssumeTrue("Test requires SortedSetDV support", DefaultCodecSupportsSortedSet());
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
FacetsConfig config = new FacetsConfig();
Document doc = new Document();
doc.Add(new SortedSetDocValuesFacetField("a", "foo1"));
writer.AddDocument(config.Build(doc));
writer.Commit();
doc = new Document();
doc.Add(new SortedSetDocValuesFacetField("a", "foo2"));
writer.AddDocument(config.Build(doc));
// NRT open
IndexSearcher searcher = new IndexSearcher(SlowCompositeReaderWrapper.Wrap(writer.Reader));
// Per-top-reader state:
SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.IndexReader);
FacetsCollector c = new FacetsCollector();
searcher.Search(new MatchAllDocsQuery(), c);
Facets facets = new SortedSetDocValuesFacetCounts(state, c);
// Ask for top 10 labels for any dims that have counts:
Assert.AreEqual("dim=a path=[] value=2 childCount=2\n foo1 (1)\n foo2 (1)\n", facets.GetTopChildren(10, "a").ToString());
IOUtils.Close(writer, searcher.IndexReader, dir);
}
public virtual void TestEmptyBucketWithMoreDocs()
{
// this test checks the logic of nextDoc() when all sub scorers have docs
// beyond the first bucket (for example). Currently, the code relies on the
// 'more' variable to work properly, and this test ensures that if the logic
// changes, we have a test to back it up.
Directory directory = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, directory);
writer.Commit();
IndexReader ir = writer.GetReader();
writer.Dispose();
IndexSearcher searcher = NewSearcher(ir);
BooleanWeight weight = (BooleanWeight)(new BooleanQuery()).CreateWeight(searcher);
BulkScorer[] scorers = new BulkScorer[] {
new BulkScorerAnonymousInnerClassHelper()
};
BooleanScorer bs = new BooleanScorer(weight, false, 1, scorers, Collections.EmptyList <BulkScorer>(), scorers.Length);
IList <int> hits = new List <int>();
bs.Score(new CollectorAnonymousInnerClassHelper(this, hits));
Assert.AreEqual(1, hits.Count, "should have only 1 hit");
Assert.AreEqual(3000, (int)hits[0], "hit should have been docID=3000");
ir.Dispose();
directory.Dispose();
}
public virtual void TestEvilSearcherFactory()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random, dir);
w.Commit();
IndexReader other = DirectoryReader.Open(dir);
SearcherFactory theEvilOne = new SearcherFactoryAnonymousClass2(this, other);
try
{
new SearcherManager(w.IndexWriter, false, theEvilOne);
fail("didn't hit expected exception");
}
catch (Exception ise) when(ise.IsIllegalStateException())
{
// expected
}
w.Dispose();
other.Dispose();
dir.Dispose();
}
public virtual void TestEvilSearcherFactory()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir);
w.Commit();
IndexReader other = DirectoryReader.Open(dir);
SearcherFactory theEvilOne = new SearcherFactoryAnonymousInnerClassHelper2(this, other);
try
{
new SearcherManager(w.w, false, theEvilOne);
Assert.Fail("didn't hit expected exception");
}
catch (InvalidOperationException ise)
{
// expected
}
w.Dispose();
other.Dispose();
dir.Dispose();
}
public virtual void TestEvilSearcherFactory()
{
Random random = Random;
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
random, dir);
w.Commit();
IndexReader other = DirectoryReader.Open(dir);
SearcherFactory theEvilOne = new SearcherFactoryAnonymousClass3(this, other);
try
{
new SearcherManager(dir, theEvilOne);
}
catch (Exception ise) when(ise.IsIllegalStateException())
{
// expected
}
try
{
new SearcherManager(w.IndexWriter, random.NextBoolean(), theEvilOne);
}
catch (Exception ise) when(ise.IsIllegalStateException())
{
// expected
}
w.Dispose();
other.Dispose();
dir.Dispose();
}
public void TestInsideBooleanQuery()
{
const string idField = "id";
const string toField = "productId";
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random, dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMergePolicy(NewLogMergePolicy()));
// 0
Document doc = new Document();
doc.Add(new TextField("description", "random text", Field.Store.NO));
doc.Add(new TextField("name", "name1", Field.Store.NO));
doc.Add(new TextField(idField, "7", Field.Store.NO));
w.AddDocument(doc);
// 1
doc = new Document();
doc.Add(new TextField("price", "10.0", Field.Store.NO));
doc.Add(new TextField(idField, "2", Field.Store.NO));
doc.Add(new TextField(toField, "7", Field.Store.NO));
w.AddDocument(doc);
// 2
doc = new Document();
doc.Add(new TextField("price", "20.0", Field.Store.NO));
doc.Add(new TextField(idField, "3", Field.Store.NO));
doc.Add(new TextField(toField, "7", Field.Store.NO));
w.AddDocument(doc);
// 3
doc = new Document();
doc.Add(new TextField("description", "more random text", Field.Store.NO));
doc.Add(new TextField("name", "name2", Field.Store.NO));
doc.Add(new TextField(idField, "0", Field.Store.NO));
w.AddDocument(doc);
w.Commit();
// 4
doc = new Document();
doc.Add(new TextField("price", "10.0", Field.Store.NO));
doc.Add(new TextField(idField, "5", Field.Store.NO));
doc.Add(new TextField(toField, "0", Field.Store.NO));
w.AddDocument(doc);
// 5
doc = new Document();
doc.Add(new TextField("price", "20.0", Field.Store.NO));
doc.Add(new TextField(idField, "6", Field.Store.NO));
doc.Add(new TextField(toField, "0", Field.Store.NO));
w.AddDocument(doc);
w.ForceMerge(1);
IndexSearcher indexSearcher = new IndexSearcher(w.GetReader());
w.Dispose();
// Search for product
Query joinQuery = JoinUtil.CreateJoinQuery(idField, false, toField,
new TermQuery(new Term("description", "random")), indexSearcher, ScoreMode.Avg);
BooleanQuery bq = new BooleanQuery();
bq.Add(joinQuery, Occur.SHOULD);
bq.Add(new TermQuery(new Term("id", "3")), Occur.SHOULD);
indexSearcher.Search(bq, new CollectorAnonymousClass());
indexSearcher.IndexReader.Dispose();
dir.Dispose();
}
public virtual void TestCommitThreadSafety()
{
const int NUM_THREADS = 5;
const double RUN_SEC = 0.5;
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));
TestUtil.ReduceOpenFiles(w.w);
w.Commit();
AtomicBoolean failed = new AtomicBoolean();
ThreadClass[] threads = new ThreadClass[NUM_THREADS];
long endTime = DateTime.Now.Millisecond + ((long)(RUN_SEC * 1000));
for (int i = 0; i < NUM_THREADS; i++)
{
int finalI = i;
threads[i] = new ThreadAnonymousInnerClassHelper(this, dir, w, failed, endTime, finalI);
threads[i].Start();
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Join();
}
Assert.IsFalse(failed.Get());
w.Dispose();
dir.Dispose();
}
private void DoTest(FieldInfo.DocValuesType_e type)
{
Directory d = NewDirectory();
IndexWriterConfig iwConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
int nDocs = AtLeast(50);
Field id = new NumericDocValuesField("id", 0);
Field f;
switch (type)
{
case FieldInfo.DocValuesType_e.BINARY:
f = new BinaryDocValuesField("dv", new BytesRef());
break;
case FieldInfo.DocValuesType_e.SORTED:
f = new SortedDocValuesField("dv", new BytesRef());
break;
case FieldInfo.DocValuesType_e.NUMERIC:
f = new NumericDocValuesField("dv", 0);
break;
default:
throw new InvalidOperationException();
}
Document document = new Document();
document.Add(id);
document.Add(f);
object[] vals = new object[nDocs];
RandomIndexWriter iw = new RandomIndexWriter(Random(), d, iwConfig);
for (int i = 0; i < nDocs; ++i)
{
id.LongValue = i;
switch (type)
{
case FieldInfo.DocValuesType_e.SORTED:
case FieldInfo.DocValuesType_e.BINARY:
do
{
vals[i] = TestUtil.RandomSimpleString(Random(), 20);
} while (((string)vals[i]).Length == 0);
f.BytesValue = new BytesRef((string)vals[i]);
break;
case FieldInfo.DocValuesType_e.NUMERIC:
int bitsPerValue = Random().NextIntBetween(1, 31); // keep it an int
vals[i] = (long)Random().Next((int)PackedInts.MaxValue(bitsPerValue));
f.LongValue = (long) vals[i];
break;
}
iw.AddDocument(document);
if (Random().NextBoolean() && i % 10 == 9)
{
iw.Commit();
}
}
iw.Dispose();
DirectoryReader rd = DirectoryReader.Open(d);
foreach (AtomicReaderContext leave in rd.Leaves)
{
FunctionValues ids = (new LongFieldSource("id")).GetValues(null, leave);
ValueSource vs;
switch (type)
{
case FieldInfo.DocValuesType_e.BINARY:
case FieldInfo.DocValuesType_e.SORTED:
vs = new BytesRefFieldSource("dv");
break;
case FieldInfo.DocValuesType_e.NUMERIC:
vs = new LongFieldSource("dv");
break;
default:
throw new InvalidOperationException();
}
FunctionValues values = vs.GetValues(null, leave);
BytesRef bytes = new BytesRef();
for (int i = 0; i < leave.AtomicReader.MaxDoc; ++i)
{
assertTrue(values.Exists(i));
if (vs is BytesRefFieldSource)
{
assertTrue(values.ObjectVal(i) is string);
}
else if (vs is LongFieldSource)
{
assertTrue(values.ObjectVal(i) is long?);
assertTrue(values.BytesVal(i, bytes));
}
else
{
throw new InvalidOperationException();
}
object expected = vals[ids.IntVal(i)];
switch (type)
{
case FieldInfo.DocValuesType_e.SORTED:
values.OrdVal(i); // no exception
assertTrue(values.NumOrd() >= 1);
goto case FieldInfo.DocValuesType_e.BINARY;
case FieldInfo.DocValuesType_e.BINARY:
assertEquals(expected, values.ObjectVal(i));
assertEquals(expected, values.StrVal(i));
assertEquals(expected, values.ObjectVal(i));
assertEquals(expected, values.StrVal(i));
assertTrue(values.BytesVal(i, bytes));
assertEquals(new BytesRef((string)expected), bytes);
break;
case FieldInfo.DocValuesType_e.NUMERIC:
assertEquals(Number.ToInt64(expected.ToString()), values.LongVal(i));
break;
}
}
}
rd.Dispose();
d.Dispose();
}
// [Test] // LUCENENET NOTE: For now, we are overriding this test in every subclass to pull it into the right context for the subclass
public virtual void TestMerge()
{
RandomDocumentFactory docFactory = new RandomDocumentFactory(this, 5, 20);
int numDocs = AtLeast(100);
int numDeletes = Random().Next(numDocs);
HashSet<int?> deletes = new HashSet<int?>();
while (deletes.Count < numDeletes)
{
deletes.Add(Random().Next(numDocs));
}
foreach (Options options in ValidOptions())
{
RandomDocument[] docs = new RandomDocument[numDocs];
for (int i = 0; i < numDocs; ++i)
{
docs[i] = docFactory.NewDocument(TestUtil.NextInt(Random(), 1, 3), AtLeast(10), options);
}
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, ClassEnvRule.Similarity, ClassEnvRule.TimeZone);
for (int i = 0; i < numDocs; ++i)
{
writer.AddDocument(AddId(docs[i].ToDocument(), "" + i));
if (Rarely())
{
writer.Commit();
}
}
foreach (int delete in deletes)
{
writer.DeleteDocuments(new Term("id", "" + delete));
}
// merge with deletes
writer.ForceMerge(1);
IndexReader reader = writer.Reader;
for (int i = 0; i < numDocs; ++i)
{
if (!deletes.Contains(i))
{
int docID = DocID(reader, "" + i);
AssertEquals(docs[i], reader.GetTermVectors(docID));
}
}
reader.Dispose();
writer.Dispose();
dir.Dispose();
}
}
public virtual void TestLocksBlock()
{
Directory src = NewDirectory();
RandomIndexWriter w1 = new RandomIndexWriter(Random(), src, Similarity, TimeZone);
w1.AddDocument(new Document());
w1.Commit();
Directory dest = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetWriteLockTimeout(1);
RandomIndexWriter w2 = new RandomIndexWriter(Random(), dest, iwc);
try
{
w2.AddIndexes(src);
Assert.Fail("did not hit expected exception");
}
catch (LockObtainFailedException lofe)
{
// expected
}
IOUtils.Close(w1, w2, src, dest);
}
public void TestEmptyDocs()
{
Directory dir = NewDirectory();
IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwConf.SetMaxBufferedDocs(RandomInts.NextIntBetween(Random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConf);
// make sure that the fact that documents might be empty is not a problem
Document emptyDoc = new Document();
int numDocs = Random().NextBoolean() ? 1 : AtLeast(1000);
for (int i = 0; i < numDocs; ++i)
{
iw.AddDocument(emptyDoc);
}
iw.Commit();
DirectoryReader rd = DirectoryReader.Open(dir);
for (int i = 0; i < numDocs; ++i)
{
Document doc = rd.Document(i);
Assert.IsNotNull(doc);
Assert.IsTrue(doc.Fields.Count <= 0);
}
rd.Dispose();
iw.Dispose();
dir.Dispose();
}
public override void SetUp()
{
base.SetUp();
AllSortFields = new List<SortField>(Arrays.AsList(new SortField[] { new SortField("byte", SortField.Type_e.BYTE, false), new SortField("short", SortField.Type_e.SHORT, false), new SortField("int", SortField.Type_e.INT, false), new SortField("long", SortField.Type_e.LONG, false), new SortField("float", SortField.Type_e.FLOAT, false), new SortField("double", SortField.Type_e.DOUBLE, false), new SortField("bytes", SortField.Type_e.STRING, false), new SortField("bytesval", SortField.Type_e.STRING_VAL, false), new SortField("byte", SortField.Type_e.BYTE, true), new SortField("short", SortField.Type_e.SHORT, true), new SortField("int", SortField.Type_e.INT, true), new SortField("long", SortField.Type_e.LONG, true), new SortField("float", SortField.Type_e.FLOAT, true), new SortField("double", SortField.Type_e.DOUBLE, true), new SortField("bytes", SortField.Type_e.STRING, true), new SortField("bytesval", SortField.Type_e.STRING_VAL, true), SortField.FIELD_SCORE, SortField.FIELD_DOC }));
if (SupportsDocValues)
{
AllSortFields.AddRange(Arrays.AsList(new SortField[] { new SortField("intdocvalues", SortField.Type_e.INT, false), new SortField("floatdocvalues", SortField.Type_e.FLOAT, false), new SortField("sortedbytesdocvalues", SortField.Type_e.STRING, false), new SortField("sortedbytesdocvaluesval", SortField.Type_e.STRING_VAL, false), new SortField("straightbytesdocvalues", SortField.Type_e.STRING_VAL, false), new SortField("intdocvalues", SortField.Type_e.INT, true), new SortField("floatdocvalues", SortField.Type_e.FLOAT, true), new SortField("sortedbytesdocvalues", SortField.Type_e.STRING, true), new SortField("sortedbytesdocvaluesval", SortField.Type_e.STRING_VAL, true), new SortField("straightbytesdocvalues", SortField.Type_e.STRING_VAL, true) }));
}
// Also test missing first / last for the "string" sorts:
foreach (string field in new string[] { "bytes", "sortedbytesdocvalues" })
{
for (int rev = 0; rev < 2; rev++)
{
bool reversed = rev == 0;
SortField sf = new SortField(field, SortField.Type_e.STRING, reversed);
sf.MissingValue = SortField.STRING_FIRST;
AllSortFields.Add(sf);
sf = new SortField(field, SortField.Type_e.STRING, reversed);
sf.MissingValue = SortField.STRING_LAST;
AllSortFields.Add(sf);
}
}
int limit = AllSortFields.Count;
for (int i = 0; i < limit; i++)
{
SortField sf = AllSortFields[i];
if (sf.Type == SortField.Type_e.INT)
{
SortField sf2 = new SortField(sf.Field, SortField.Type_e.INT, sf.Reverse);
sf2.MissingValue = Random().Next();
AllSortFields.Add(sf2);
}
else if (sf.Type == SortField.Type_e.LONG)
{
SortField sf2 = new SortField(sf.Field, SortField.Type_e.LONG, sf.Reverse);
sf2.MissingValue = Random().NextLong();
AllSortFields.Add(sf2);
}
else if (sf.Type == SortField.Type_e.FLOAT)
{
SortField sf2 = new SortField(sf.Field, SortField.Type_e.FLOAT, sf.Reverse);
sf2.MissingValue = (float)Random().NextDouble();
AllSortFields.Add(sf2);
}
else if (sf.Type == SortField.Type_e.DOUBLE)
{
SortField sf2 = new SortField(sf.Field, SortField.Type_e.DOUBLE, sf.Reverse);
sf2.MissingValue = Random().NextDouble();
AllSortFields.Add(sf2);
}
}
Dir = NewDirectory();
RandomIndexWriter iw = new RandomIndexWriter(Random(), Dir);
int numDocs = AtLeast(200);
for (int i = 0; i < numDocs; i++)
{
IList<Field> fields = new List<Field>();
fields.Add(NewTextField("english", English.IntToEnglish(i), Field.Store.NO));
fields.Add(NewTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
fields.Add(NewStringField("byte", "" + ((sbyte)Random().Next()), Field.Store.NO));
fields.Add(NewStringField("short", "" + ((short)Random().Next()), Field.Store.NO));
fields.Add(new IntField("int", Random().Next(), Field.Store.NO));
fields.Add(new LongField("long", Random().NextLong(), Field.Store.NO));
fields.Add(new FloatField("float", (float)Random().NextDouble(), Field.Store.NO));
fields.Add(new DoubleField("double", Random().NextDouble(), Field.Store.NO));
fields.Add(NewStringField("bytes", TestUtil.RandomRealisticUnicodeString(Random()), Field.Store.NO));
fields.Add(NewStringField("bytesval", TestUtil.RandomRealisticUnicodeString(Random()), Field.Store.NO));
fields.Add(new DoubleField("double", Random().NextDouble(), Field.Store.NO));
if (SupportsDocValues)
{
fields.Add(new NumericDocValuesField("intdocvalues", Random().Next()));
fields.Add(new FloatDocValuesField("floatdocvalues", (float)Random().NextDouble()));
fields.Add(new SortedDocValuesField("sortedbytesdocvalues", new BytesRef(TestUtil.RandomRealisticUnicodeString(Random()))));
fields.Add(new SortedDocValuesField("sortedbytesdocvaluesval", new BytesRef(TestUtil.RandomRealisticUnicodeString(Random()))));
fields.Add(new BinaryDocValuesField("straightbytesdocvalues", new BytesRef(TestUtil.RandomRealisticUnicodeString(Random()))));
}
Document document = new Document();
document.Add(new StoredField("id", "" + i));
if (VERBOSE)
{
Console.WriteLine(" add doc id=" + i);
}
foreach (Field field in fields)
{
// So we are sometimes missing that field:
if (Random().Next(5) != 4)
{
document.Add(field);
if (VERBOSE)
{
Console.WriteLine(" " + field);
}
}
}
iw.AddDocument(document);
if (Random().Next(50) == 17)
{
iw.Commit();
}
}
Reader = iw.Reader;
iw.Dispose();
Searcher = NewSearcher(Reader);
if (VERBOSE)
{
Console.WriteLine(" searcher=" + Searcher);
}
}
public void TestBigDocuments()
{
// "big" as "much bigger than the chunk size"
// for this test we force a FS dir
// we can't just use newFSDirectory, because this test doesn't really index anything.
// so if we get NRTCachingDir+SimpleText, we make massive stored fields and OOM (LUCENE-4484)
Directory dir = new MockDirectoryWrapper(Random(), new MMapDirectory(CreateTempDir("testBigDocuments")));
IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwConf.SetMaxBufferedDocs(RandomInts.NextIntBetween(Random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConf);
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
Document emptyDoc = new Document(); // emptyDoc
Document bigDoc1 = new Document(); // lot of small fields
Document bigDoc2 = new Document(); // 1 very big field
Field idField = new StringField("id", "", Field.Store.NO);
emptyDoc.Add(idField);
bigDoc1.Add(idField);
bigDoc2.Add(idField);
FieldType onlyStored = new FieldType(StringField.TYPE_STORED);
onlyStored.Indexed = false;
Field smallField = new Field("fld", RandomByteArray(Random().Next(10), 256), onlyStored);
int numFields = RandomInts.NextIntBetween(Random(), 500000, 1000000);
for (int i = 0; i < numFields; ++i)
{
bigDoc1.Add(smallField);
}
Field bigField = new Field("fld", RandomByteArray(RandomInts.NextIntBetween(Random(), 1000000, 5000000), 2), onlyStored);
bigDoc2.Add(bigField);
int numDocs = AtLeast(5);
Document[] docs = new Document[numDocs];
for (int i = 0; i < numDocs; ++i)
{
docs[i] = RandomInts.RandomFrom(Random(), Arrays.AsList(emptyDoc, bigDoc1, bigDoc2));
}
for (int i = 0; i < numDocs; ++i)
{
idField.StringValue = "" + i;
iw.AddDocument(docs[i]);
if (Random().Next(numDocs) == 0)
{
iw.Commit();
}
}
iw.Commit();
iw.ForceMerge(1); // look at what happens when big docs are merged
DirectoryReader rd = DirectoryReader.Open(dir);
IndexSearcher searcher = new IndexSearcher(rd);
for (int i = 0; i < numDocs; ++i)
{
Query query = new TermQuery(new Term("id", "" + i));
TopDocs topDocs = searcher.Search(query, 1);
Assert.AreEqual(1, topDocs.TotalHits, "" + i);
Document doc = rd.Document(topDocs.ScoreDocs[0].Doc);
Assert.IsNotNull(doc);
IndexableField[] fieldValues = doc.GetFields("fld");
Assert.AreEqual(docs[i].GetFields("fld").Length, fieldValues.Length);
if (fieldValues.Length > 0)
{
Assert.AreEqual(docs[i].GetFields("fld")[0].BinaryValue(), fieldValues[0].BinaryValue());
}
}
rd.Dispose();
iw.Dispose();
dir.Dispose();
}
public void TestConcurrentReads()
{
Directory dir = NewDirectory();
IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwConf.SetMaxBufferedDocs(RandomInts.NextIntBetween(Random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConf);
// make sure the readers are properly cloned
Document doc = new Document();
Field field = new StringField("fld", "", Field.Store.YES);
doc.Add(field);
int numDocs = AtLeast(1000);
for (int i = 0; i < numDocs; ++i)
{
field.StringValue = "" + i;
iw.AddDocument(doc);
}
iw.Commit();
DirectoryReader rd = DirectoryReader.Open(dir);
IndexSearcher searcher = new IndexSearcher(rd);
int concurrentReads = AtLeast(5);
int readsPerThread = AtLeast(50);
IList<ThreadClass> readThreads = new List<ThreadClass>();
AtomicReference<Exception> ex = new AtomicReference<Exception>();
for (int i = 0; i < concurrentReads; ++i)
{
readThreads.Add(new ThreadAnonymousInnerClassHelper(numDocs, rd, searcher, readsPerThread, ex, i));
}
foreach (ThreadClass thread in readThreads)
{
thread.Start();
}
foreach (ThreadClass thread in readThreads)
{
thread.Join();
}
rd.Dispose();
if (ex.Value != null)
{
throw ex.Value;
}
iw.Dispose();
dir.Dispose();
}
public void TestSimpleWithScoring()
{
const string idField = "id";
const string toField = "movieId";
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random, dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMergePolicy(NewLogMergePolicy()));
// 0
Document doc = new Document();
doc.Add(new TextField("description", "A random movie", Field.Store.NO));
doc.Add(new TextField("name", "Movie 1", Field.Store.NO));
doc.Add(new TextField(idField, "1", Field.Store.NO));
w.AddDocument(doc);
// 1
doc = new Document();
doc.Add(new TextField("subtitle", "The first subtitle of this movie", Field.Store.NO));
doc.Add(new TextField(idField, "2", Field.Store.NO));
doc.Add(new TextField(toField, "1", Field.Store.NO));
w.AddDocument(doc);
// 2
doc = new Document();
doc.Add(new TextField("subtitle", "random subtitle; random event movie", Field.Store.NO));
doc.Add(new TextField(idField, "3", Field.Store.NO));
doc.Add(new TextField(toField, "1", Field.Store.NO));
w.AddDocument(doc);
// 3
doc = new Document();
doc.Add(new TextField("description", "A second random movie", Field.Store.NO));
doc.Add(new TextField("name", "Movie 2", Field.Store.NO));
doc.Add(new TextField(idField, "4", Field.Store.NO));
w.AddDocument(doc);
w.Commit();
// 4
doc = new Document();
doc.Add(new TextField("subtitle", "a very random event happened during christmas night", Field.Store.NO));
doc.Add(new TextField(idField, "5", Field.Store.NO));
doc.Add(new TextField(toField, "4", Field.Store.NO));
w.AddDocument(doc);
// 5
doc = new Document();
doc.Add(new TextField("subtitle", "movie end movie test 123 test 123 random", Field.Store.NO));
doc.Add(new TextField(idField, "6", Field.Store.NO));
doc.Add(new TextField(toField, "4", Field.Store.NO));
w.AddDocument(doc);
IndexSearcher indexSearcher = new IndexSearcher(w.GetReader());
w.Dispose();
// Search for movie via subtitle
Query joinQuery = JoinUtil.CreateJoinQuery(toField, false, idField,
new TermQuery(new Term("subtitle", "random")), indexSearcher, ScoreMode.Max);
TopDocs result = indexSearcher.Search(joinQuery, 10);
assertEquals(2, result.TotalHits);
assertEquals(0, result.ScoreDocs[0].Doc);
assertEquals(3, result.ScoreDocs[1].Doc);
// Score mode max.
joinQuery = JoinUtil.CreateJoinQuery(toField, false, idField, new TermQuery(new Term("subtitle", "movie")),
indexSearcher, ScoreMode.Max);
result = indexSearcher.Search(joinQuery, 10);
assertEquals(2, result.TotalHits);
assertEquals(3, result.ScoreDocs[0].Doc);
assertEquals(0, result.ScoreDocs[1].Doc);
// Score mode total
joinQuery = JoinUtil.CreateJoinQuery(toField, false, idField, new TermQuery(new Term("subtitle", "movie")),
indexSearcher, ScoreMode.Total);
result = indexSearcher.Search(joinQuery, 10);
assertEquals(2, result.TotalHits);
assertEquals(0, result.ScoreDocs[0].Doc);
assertEquals(3, result.ScoreDocs[1].Doc);
//Score mode avg
joinQuery = JoinUtil.CreateJoinQuery(toField, false, idField, new TermQuery(new Term("subtitle", "movie")),
indexSearcher, ScoreMode.Avg);
result = indexSearcher.Search(joinQuery, 10);
assertEquals(2, result.TotalHits);
assertEquals(3, result.ScoreDocs[0].Doc);
assertEquals(0, result.ScoreDocs[1].Doc);
indexSearcher.IndexReader.Dispose();
dir.Dispose();
}
public void TestNestedDocScoringWithDeletes()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.COMPOUND_FILES));
// Cannot assert this since we use NoMergePolicy:
w.DoRandomForceMergeAssert = false;
IList<Document> docs = new List<Document>();
docs.Add(MakeJob("java", 2007));
docs.Add(MakeJob("python", 2010));
docs.Add(MakeResume("Lisa", "United Kingdom"));
w.AddDocuments(docs);
docs.Clear();
docs.Add(MakeJob("c", 1999));
docs.Add(MakeJob("ruby", 2005));
docs.Add(MakeJob("java", 2006));
docs.Add(MakeResume("Frank", "United States"));
w.AddDocuments(docs);
w.Commit();
IndexSearcher s = NewSearcher(DirectoryReader.Open(dir));
ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(NumericRangeQuery.NewIntRange("year", 1990, 2010, true, true), new FixedBitSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("docType", "resume")))), ScoreMode.Total);
TopDocs topDocs = s.Search(q, 10);
assertEquals(2, topDocs.TotalHits);
assertEquals(6, topDocs.ScoreDocs[0].Doc);
assertEquals(3.0f, topDocs.ScoreDocs[0].Score, 0.0f);
assertEquals(2, topDocs.ScoreDocs[1].Doc);
assertEquals(2.0f, topDocs.ScoreDocs[1].Score, 0.0f);
s.IndexReader.Dispose();
w.DeleteDocuments(new Term("skill", "java"));
w.Dispose();
s = NewSearcher(DirectoryReader.Open(dir));
topDocs = s.Search(q, 10);
assertEquals(2, topDocs.TotalHits);
assertEquals(6, topDocs.ScoreDocs[0].Doc);
assertEquals(2.0f, topDocs.ScoreDocs[0].Score, 0.0f);
assertEquals(2, topDocs.ScoreDocs[1].Doc);
assertEquals(1.0f, topDocs.ScoreDocs[1].Score, 0.0f);
s.IndexReader.Dispose();
dir.Dispose();
}
public void TestBulkMergeWithDeletes()
{
int numDocs = AtLeast(200);
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.COMPOUND_FILES));
for (int i = 0; i < numDocs; ++i)
{
Document doc = new Document();
doc.Add(new StringField("id", Convert.ToString(i), Field.Store.YES));
doc.Add(new StoredField("f", TestUtil.RandomSimpleString(Random())));
w.AddDocument(doc);
}
int deleteCount = TestUtil.NextInt(Random(), 5, numDocs);
for (int i = 0; i < deleteCount; ++i)
{
int id = Random().Next(numDocs);
w.DeleteDocuments(new Term("id", Convert.ToString(id)));
}
w.Commit();
w.Dispose();
w = new RandomIndexWriter(Random(), dir);
w.ForceMerge(TestUtil.NextInt(Random(), 1, 3));
w.Commit();
w.Dispose();
TestUtil.CheckIndex(dir);
dir.Dispose();
}
public void TestSimple()
{
const string idField = "id";
const string toField = "productId";
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random, dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMergePolicy(NewLogMergePolicy()));
// 0
Document doc = new Document();
doc.Add(new TextField("description", "random text", Field.Store.NO));
doc.Add(new TextField("name", "name1", Field.Store.NO));
doc.Add(new TextField(idField, "1", Field.Store.NO));
w.AddDocument(doc);
// 1
doc = new Document();
doc.Add(new TextField("price", "10.0", Field.Store.NO));
doc.Add(new TextField(idField, "2", Field.Store.NO));
doc.Add(new TextField(toField, "1", Field.Store.NO));
w.AddDocument(doc);
// 2
doc = new Document();
doc.Add(new TextField("price", "20.0", Field.Store.NO));
doc.Add(new TextField(idField, "3", Field.Store.NO));
doc.Add(new TextField(toField, "1", Field.Store.NO));
w.AddDocument(doc);
// 3
doc = new Document();
doc.Add(new TextField("description", "more random text", Field.Store.NO));
doc.Add(new TextField("name", "name2", Field.Store.NO));
doc.Add(new TextField(idField, "4", Field.Store.NO));
w.AddDocument(doc);
w.Commit();
// 4
doc = new Document();
doc.Add(new TextField("price", "10.0", Field.Store.NO));
doc.Add(new TextField(idField, "5", Field.Store.NO));
doc.Add(new TextField(toField, "4", Field.Store.NO));
w.AddDocument(doc);
// 5
doc = new Document();
doc.Add(new TextField("price", "20.0", Field.Store.NO));
doc.Add(new TextField(idField, "6", Field.Store.NO));
doc.Add(new TextField(toField, "4", Field.Store.NO));
w.AddDocument(doc);
IndexSearcher indexSearcher = new IndexSearcher(w.GetReader());
w.Dispose();
// Search for product
Query joinQuery = JoinUtil.CreateJoinQuery(idField, false, toField, new TermQuery(new Term("name", "name2")),
indexSearcher, ScoreMode.None);
TopDocs result = indexSearcher.Search(joinQuery, 10);
assertEquals(2, result.TotalHits);
assertEquals(4, result.ScoreDocs[0].Doc);
assertEquals(5, result.ScoreDocs[1].Doc);
joinQuery = JoinUtil.CreateJoinQuery(idField, false, toField, new TermQuery(new Term("name", "name1")),
indexSearcher, ScoreMode.None);
result = indexSearcher.Search(joinQuery, 10);
assertEquals(2, result.TotalHits);
assertEquals(1, result.ScoreDocs[0].Doc);
assertEquals(2, result.ScoreDocs[1].Doc);
// Search for offer
joinQuery = JoinUtil.CreateJoinQuery(toField, false, idField, new TermQuery(new Term("id", "5")),
indexSearcher, ScoreMode.None);
result = indexSearcher.Search(joinQuery, 10);
assertEquals(1, result.TotalHits);
assertEquals(3, result.ScoreDocs[0].Doc);
indexSearcher.IndexReader.Dispose();
dir.Dispose();
}
public void TestTotalGroupCount()
{
string groupField = "author";
FieldType customType = new FieldType();
customType.Stored = true;
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
Random(),
dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));
bool canUseIDV = !"Lucene3x".Equals(w.w.Config.Codec.Name, StringComparison.Ordinal);
// 0
Document doc = new Document();
AddGroupField(doc, groupField, "author1", canUseIDV);
doc.Add(new TextField("content", "random text", Field.Store.YES));
doc.Add(new Field("id", "1", customType));
w.AddDocument(doc);
// 1
doc = new Document();
AddGroupField(doc, groupField, "author1", canUseIDV);
doc.Add(new TextField("content", "some more random text blob", Field.Store.YES));
doc.Add(new Field("id", "2", customType));
w.AddDocument(doc);
// 2
doc = new Document();
AddGroupField(doc, groupField, "author1", canUseIDV);
doc.Add(new TextField("content", "some more random textual data", Field.Store.YES));
doc.Add(new Field("id", "3", customType));
w.AddDocument(doc);
w.Commit(); // To ensure a second segment
// 3
doc = new Document();
AddGroupField(doc, groupField, "author2", canUseIDV);
doc.Add(new TextField("content", "some random text", Field.Store.YES));
doc.Add(new Field("id", "4", customType));
w.AddDocument(doc);
// 4
doc = new Document();
AddGroupField(doc, groupField, "author3", canUseIDV);
doc.Add(new TextField("content", "some more random text", Field.Store.YES));
doc.Add(new Field("id", "5", customType));
w.AddDocument(doc);
// 5
doc = new Document();
AddGroupField(doc, groupField, "author3", canUseIDV);
doc.Add(new TextField("content", "random blob", Field.Store.YES));
doc.Add(new Field("id", "6", customType));
w.AddDocument(doc);
// 6 -- no author field
doc = new Document();
doc.Add(new TextField("content", "random word stuck in alot of other text", Field.Store.YES));
doc.Add(new Field("id", "6", customType));
w.AddDocument(doc);
IndexSearcher indexSearcher = NewSearcher(w.Reader);
w.Dispose();
IAbstractAllGroupsCollector<object> allGroupsCollector = CreateRandomCollector(groupField, canUseIDV);
// LUCENENET TODO: Create an ICollector interface that we can inherit our Collector interfaces from
// so this cast is not necessary. Consider eliminating the Collector abstract class.
indexSearcher.Search(new TermQuery(new Term("content", "random")), allGroupsCollector as Collector);
assertEquals(4, allGroupsCollector.GroupCount);
allGroupsCollector = CreateRandomCollector(groupField, canUseIDV);
// LUCENENET TODO: Create an ICollector interface that we can inherit our Collector interfaces from
// so this cast is not necessary. Consider eliminating the Collector abstract class.
indexSearcher.Search(new TermQuery(new Term("content", "some")), allGroupsCollector as Collector);
assertEquals(3, allGroupsCollector.GroupCount);
allGroupsCollector = CreateRandomCollector(groupField, canUseIDV);
// LUCENENET TODO: Create an ICollector interface that we can inherit our Collector interfaces from
// so this cast is not necessary. Consider eliminating the Collector abstract class.
indexSearcher.Search(new TermQuery(new Term("content", "blob")), allGroupsCollector as Collector);
assertEquals(2, allGroupsCollector.GroupCount);
indexSearcher.IndexReader.Dispose();
dir.Dispose();
}
public virtual void TestTermUTF16SortOrder()
{
Random rnd = Random();
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(rnd, dir, Similarity, TimeZone);
Document d = new Document();
// Single segment
Field f = NewStringField("f", "", Field.Store.NO);
d.Add(f);
char[] chars = new char[2];
HashSet<string> allTerms = new HashSet<string>();
int num = AtLeast(200);
for (int i = 0; i < num; i++)
{
string s;
if (rnd.NextBoolean())
{
// Single char
if (rnd.NextBoolean())
{
// Above surrogates
chars[0] = (char)GetInt(rnd, 1 + UnicodeUtil.UNI_SUR_LOW_END, 0xffff);
}
else
{
// Below surrogates
chars[0] = (char)GetInt(rnd, 0, UnicodeUtil.UNI_SUR_HIGH_START - 1);
}
s = new string(chars, 0, 1);
}
else
{
// Surrogate pair
chars[0] = (char)GetInt(rnd, UnicodeUtil.UNI_SUR_HIGH_START, UnicodeUtil.UNI_SUR_HIGH_END);
Assert.IsTrue(((int)chars[0]) >= UnicodeUtil.UNI_SUR_HIGH_START && ((int)chars[0]) <= UnicodeUtil.UNI_SUR_HIGH_END);
chars[1] = (char)GetInt(rnd, UnicodeUtil.UNI_SUR_LOW_START, UnicodeUtil.UNI_SUR_LOW_END);
s = new string(chars, 0, 2);
}
allTerms.Add(s);
f.StringValue = s;
writer.AddDocument(d);
if ((1 + i) % 42 == 0)
{
writer.Commit();
}
}
IndexReader r = writer.Reader;
// Test each sub-segment
foreach (AtomicReaderContext ctx in r.Leaves)
{
CheckTermsOrder(ctx.Reader, allTerms, false);
}
CheckTermsOrder(r, allTerms, true);
// Test multi segment
r.Dispose();
writer.ForceMerge(1);
// Test single segment
r = writer.Reader;
CheckTermsOrder(r, allTerms, true);
r.Dispose();
writer.Dispose();
dir.Dispose();
}
// TODO: create a testNormsNotPresent ourselves by adding/deleting/merging docs
public virtual void BuildIndex(Directory dir)
{
Random random = Random();
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig config = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
Similarity provider = new MySimProvider(this);
config.SetSimilarity(provider);
RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues());
int num = AtLeast(100);
for (int i = 0; i < num; i++)
{
Document doc = docs.NextDoc();
int boost = Random().Next(255);
Field f = new TextField(ByteTestField, "" + boost, Field.Store.YES);
f.Boost = boost;
doc.Add(f);
writer.AddDocument(doc);
doc.RemoveField(ByteTestField);
if (Rarely())
{
writer.Commit();
}
}
writer.Commit();
writer.Dispose();
docs.Dispose();
}
public void TestWithDeletions()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
KeyValuePair <List <string>, IDictionary <string, Document> > res = GenerateIndexDocuments(AtLeast(1000), false, false);
IDictionary <string, Document> docs = res.Value;
List <String> invalidDocTerms = res.Key;
Random rand = Random();
List <string> termsToDel = new List <string>();
foreach (Document doc in docs.Values)
{
IIndexableField f2 = doc.GetField(FIELD_NAME);
if (rand.nextBoolean() && f2 != null && !invalidDocTerms.Contains(f2.GetStringValue()))
{
termsToDel.Add(doc.Get(FIELD_NAME));
}
writer.AddDocument(doc);
}
writer.Commit();
Term[] delTerms = new Term[termsToDel.size()];
for (int i = 0; i < termsToDel.size(); i++)
{
delTerms[i] = new Term(FIELD_NAME, termsToDel[i]);
}
foreach (Term delTerm in delTerms)
{
writer.DeleteDocuments(delTerm);
}
writer.Commit();
writer.Dispose();
foreach (string termToDel in termsToDel)
{
var toDel = docs[termToDel];
assertTrue(toDel != null);
docs.Remove(termToDel);
}
IndexReader ir = DirectoryReader.Open(dir);
assertEquals(ir.NumDocs, docs.size());
IDictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
IInputIterator inputIterator = dictionary.GetEntryIterator();
BytesRef f;
while ((f = inputIterator.Next()) != null)
{
var field = f.Utf8ToString();
Document doc = docs.ContainsKey(field) ? docs[field] : null;
docs.Remove(field);
assertTrue(f.equals(new BytesRef(doc.Get(FIELD_NAME))));
IIndexableField weightField = doc.GetField(WEIGHT_FIELD_NAME);
assertEquals(inputIterator.Weight, (weightField != null) ? Convert.ToInt64(weightField.GetNumericValue()) : 0);
assertEquals(inputIterator.Payload, null);
}
foreach (string invalidTerm in invalidDocTerms)
{
var invalid = docs[invalidTerm];
docs.Remove(invalidTerm);
assertNotNull(invalid);
}
assertTrue(!docs.Any());
ir.Dispose();
dir.Dispose();
}
public void TestReadSkip()
{
Directory dir = NewDirectory();
IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwConf.SetMaxBufferedDocs(RandomInts.NextIntBetween(Random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConf);
FieldType ft = new FieldType();
ft.Stored = true;
ft.Freeze();
string @string = TestUtil.RandomSimpleString(Random(), 50);
var bytes = @string.GetBytes(IOUtils.CHARSET_UTF_8);
long l = Random().NextBoolean() ? Random().Next(42) : Random().NextLong();
int i = Random().NextBoolean() ? Random().Next(42) : Random().Next();
float f = Random().NextFloat();
double d = Random().NextDouble();
IList<Field> fields = Arrays.AsList(new Field("bytes", bytes, ft), new Field("string", @string, ft), new LongField("long", l, Field.Store.YES), new IntField("int", i, Field.Store.YES), new FloatField("float", f, Field.Store.YES), new DoubleField("double", d, Field.Store.YES)
);
for (int k = 0; k < 100; ++k)
{
Document doc = new Document();
foreach (Field fld in fields)
{
doc.Add(fld);
}
iw.w.AddDocument(doc);
}
iw.Commit();
DirectoryReader reader = DirectoryReader.Open(dir);
int docID = Random().Next(100);
foreach (Field fld in fields)
{
string fldName = fld.Name();
Document sDoc = reader.Document(docID, Collections.Singleton(fldName));
IndexableField sField = sDoc.GetField(fldName);
if (typeof(Field) == fld.GetType())
{
Assert.AreEqual(fld.BinaryValue(), sField.BinaryValue());
Assert.AreEqual(fld.StringValue, sField.StringValue);
}
else
{
Assert.AreEqual(fld.NumericValue, sField.NumericValue);
}
}
reader.Dispose();
iw.Dispose();
dir.Dispose();
}
public void TestTotalGroupCount()
{
string groupField = "author";
FieldType customType = new FieldType();
customType.IsStored = true;
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
Random,
dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(Random)).SetMergePolicy(NewLogMergePolicy()));
bool canUseIDV = !"Lucene3x".Equals(w.IndexWriter.Config.Codec.Name, StringComparison.Ordinal);
// 0
Document doc = new Document();
AddGroupField(doc, groupField, "author1", canUseIDV);
doc.Add(new TextField("content", "random text", Field.Store.YES));
doc.Add(new Field("id", "1", customType));
w.AddDocument(doc);
// 1
doc = new Document();
AddGroupField(doc, groupField, "author1", canUseIDV);
doc.Add(new TextField("content", "some more random text blob", Field.Store.YES));
doc.Add(new Field("id", "2", customType));
w.AddDocument(doc);
// 2
doc = new Document();
AddGroupField(doc, groupField, "author1", canUseIDV);
doc.Add(new TextField("content", "some more random textual data", Field.Store.YES));
doc.Add(new Field("id", "3", customType));
w.AddDocument(doc);
w.Commit(); // To ensure a second segment
// 3
doc = new Document();
AddGroupField(doc, groupField, "author2", canUseIDV);
doc.Add(new TextField("content", "some random text", Field.Store.YES));
doc.Add(new Field("id", "4", customType));
w.AddDocument(doc);
// 4
doc = new Document();
AddGroupField(doc, groupField, "author3", canUseIDV);
doc.Add(new TextField("content", "some more random text", Field.Store.YES));
doc.Add(new Field("id", "5", customType));
w.AddDocument(doc);
// 5
doc = new Document();
AddGroupField(doc, groupField, "author3", canUseIDV);
doc.Add(new TextField("content", "random blob", Field.Store.YES));
doc.Add(new Field("id", "6", customType));
w.AddDocument(doc);
// 6 -- no author field
doc = new Document();
doc.Add(new TextField("content", "random word stuck in alot of other text", Field.Store.YES));
doc.Add(new Field("id", "6", customType));
w.AddDocument(doc);
IndexSearcher indexSearcher = NewSearcher(w.GetReader());
w.Dispose();
IAbstractAllGroupsCollector <object> allGroupsCollector = CreateRandomCollector(groupField, canUseIDV);
indexSearcher.Search(new TermQuery(new Term("content", "random")), allGroupsCollector);
assertEquals(4, allGroupsCollector.GroupCount);
allGroupsCollector = CreateRandomCollector(groupField, canUseIDV);
indexSearcher.Search(new TermQuery(new Term("content", "some")), allGroupsCollector);
assertEquals(3, allGroupsCollector.GroupCount);
allGroupsCollector = CreateRandomCollector(groupField, canUseIDV);
indexSearcher.Search(new TermQuery(new Term("content", "blob")), allGroupsCollector);
assertEquals(2, allGroupsCollector.GroupCount);
indexSearcher.IndexReader.Dispose();
dir.Dispose();
}
public virtual void TestSortedSetWithDups()
{
AssumeTrue("codec does not support SORTED_SET", DefaultCodecSupportsSortedSet());
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
int numValues = Random().Next(5);
for (int j = 0; j < numValues; j++)
{
doc.Add(new SortedSetDocValuesField("bytes", new BytesRef(TestUtil.RandomSimpleString(Random(), 2))));
}
iw.AddDocument(doc);
if (Random().Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.Reader;
iw.ForceMerge(1);
DirectoryReader ir2 = iw.Reader;
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
SortedSetDocValues multi = MultiDocValues.GetSortedSetValues(ir, "bytes");
SortedSetDocValues single = merged.GetSortedSetDocValues("bytes");
if (multi == null)
{
Assert.IsNull(single);
}
else
{
Assert.AreEqual(single.ValueCount, multi.ValueCount);
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
// check values
for (long i = 0; i < single.ValueCount; i++)
{
single.LookupOrd(i, expected);
multi.LookupOrd(i, actual);
Assert.AreEqual(expected, actual);
}
// check ord list
for (int i = 0; i < numDocs; i++)
{
single.Document = i;
List<long?> expectedList = new List<long?>();
long ord;
while ((ord = single.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
expectedList.Add(ord);
}
multi.Document = i;
int upto = 0;
while ((ord = multi.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
Assert.AreEqual((long)expectedList[upto], ord);
upto++;
}
Assert.AreEqual(expectedList.Count, upto);
}
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}
public virtual void TestListAll()
{
Directory dir = NewDirectory();
// riw should sometimes create docvalues fields, etc
RandomIndexWriter riw = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
Document doc = new Document();
// these fields should sometimes get term vectors, etc
Field idField = NewStringField("id", "", Field.Store.NO);
Field bodyField = NewTextField("body", "", Field.Store.NO);
doc.Add(idField);
doc.Add(bodyField);
for (int i = 0; i < 100; i++)
{
idField.StringValue = Convert.ToString(i);
bodyField.StringValue = TestUtil.RandomUnicodeString(Random());
riw.AddDocument(doc);
if (Random().Next(7) == 0)
{
riw.Commit();
}
}
riw.Dispose();
CheckFiles(dir);
dir.Dispose();
}
public virtual void TestDocsWithField()
{
AssumeTrue("codec does not support docsWithField", DefaultCodecSupportsDocsWithField());
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
if (Random().Next(4) >= 0)
{
doc.Add(new NumericDocValuesField("numbers", Random().NextLong()));
}
doc.Add(new NumericDocValuesField("numbersAlways", Random().NextLong()));
iw.AddDocument(doc);
if (Random().Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.Reader;
iw.ForceMerge(1);
DirectoryReader ir2 = iw.Reader;
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
Bits multi = MultiDocValues.GetDocsWithField(ir, "numbers");
Bits single = merged.GetDocsWithField("numbers");
if (multi == null)
{
Assert.IsNull(single);
}
else
{
Assert.AreEqual(single.Length(), multi.Length());
for (int i = 0; i < numDocs; i++)
{
Assert.AreEqual(single.Get(i), multi.Get(i));
}
}
multi = MultiDocValues.GetDocsWithField(ir, "numbersAlways");
single = merged.GetDocsWithField("numbersAlways");
Assert.AreEqual(single.Length(), multi.Length());
for (int i = 0; i < numDocs; i++)
{
Assert.AreEqual(single.Get(i), multi.Get(i));
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}