public virtual void Test()
{
NumericDocValues fooNorms = MultiDocValues.GetNormValues(reader, "foo");
for (int i = 0; i < reader.MaxDoc; i++)
{
Assert.AreEqual((int)expected[i], fooNorms.Get(i) & 0xff);
}
}
public virtual void TestSorted()
{
Directory dir = NewDirectory();
Document doc = new Document();
BytesRef @ref = new BytesRef();
Field field = new SortedDocValuesField("bytes", @ref);
doc.Add(field);
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
@ref.CopyChars(TestUtil.RandomUnicodeString(Random()));
if (DefaultCodecSupportsDocsWithField() && Random().Next(7) == 0)
{
iw.AddDocument(new Document());
}
iw.AddDocument(doc);
if (Random().Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.Reader;
iw.ForceMerge(1);
DirectoryReader ir2 = iw.Reader;
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
SortedDocValues multi = MultiDocValues.GetSortedValues(ir, "bytes");
SortedDocValues single = merged.GetSortedDocValues("bytes");
Assert.AreEqual(single.ValueCount, multi.ValueCount);
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
for (int i = 0; i < numDocs; i++)
{
// check ord
Assert.AreEqual(single.GetOrd(i), multi.GetOrd(i));
// check value
single.Get(i, expected);
multi.Get(i, actual);
Assert.AreEqual(expected, actual);
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}
public void Test()
{
NumericDocValues fooNorms = MultiDocValues.GetNormValues(reader, "foo");
assertNotNull(fooNorms);
for (int i = 0; i < reader.MaxDoc; i++)
{
assertEquals(expected[i], fooNorms.Get(i));
}
}
/// <summary>
/// Indexes at least 1 document with f1, and at least 1 document with f2.
/// returns the norms for "field".
/// </summary>
internal virtual NumericDocValues GetNorms(string field, Field f1, Field f2)
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter riw = new RandomIndexWriter(Random, dir, iwc);
// add f1
Document d = new Document();
d.Add(f1);
riw.AddDocument(d);
// add f2
d = new Document();
d.Add(f2);
riw.AddDocument(d);
// add a mix of f1's and f2's
int numExtraDocs = TestUtil.NextInt32(Random, 1, 1000);
for (int i = 0; i < numExtraDocs; i++)
{
d = new Document();
d.Add(Random.NextBoolean() ? f1 : f2);
riw.AddDocument(d);
}
IndexReader ir1 = riw.GetReader();
// todo: generalize
NumericDocValues norms1 = MultiDocValues.GetNormValues(ir1, field);
// fully merge and validate MultiNorms against single segment.
riw.ForceMerge(1);
DirectoryReader ir2 = riw.GetReader();
NumericDocValues norms2 = GetOnlySegmentReader(ir2).GetNormValues(field);
if (norms1 == null)
{
Assert.IsNull(norms2);
}
else
{
for (int docID = 0; docID < ir1.MaxDoc; docID++)
{
Assert.AreEqual(norms1.Get(docID), norms2.Get(docID));
}
}
ir1.Dispose();
ir2.Dispose();
riw.Dispose();
dir.Dispose();
return(norms1);
}
public override SortedSetDocValues GetSortedSetDocValues(string field)
{
EnsureOpen();
OrdinalMap map = null;
UninterruptableMonitor.Enter(cachedOrdMaps);
try
{
if (!cachedOrdMaps.TryGetValue(field, out map))
{
// uncached, or not a multi dv
SortedSetDocValues dv = MultiDocValues.GetSortedSetValues(@in, field);
if (dv is MultiSortedSetDocValues docValues)
{
map = docValues.Mapping;
if (map.owner == CoreCacheKey)
{
cachedOrdMaps[field] = map;
}
}
return(dv);
}
}
finally
{
UninterruptableMonitor.Exit(cachedOrdMaps);
}
// cached ordinal map
if (FieldInfos.FieldInfo(field).DocValuesType != DocValuesType.SORTED_SET)
{
return(null);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(map != null);
}
int size = @in.Leaves.Count;
var values = new SortedSetDocValues[size];
int[] starts = new int[size + 1];
for (int i = 0; i < size; i++)
{
AtomicReaderContext context = @in.Leaves[i];
SortedSetDocValues v = context.AtomicReader.GetSortedSetDocValues(field) ?? DocValues.EMPTY_SORTED_SET;
values[i] = v;
starts[i] = context.DocBase;
}
starts[size] = MaxDoc;
return(new MultiSortedSetDocValues(values, starts, map));
}
public virtual void TestBinary()
{
Directory dir = NewDirectory();
Document doc = new Document();
BytesRef @ref = new BytesRef();
Field field = new BinaryDocValuesField("bytes", @ref);
doc.Add(field);
IndexWriterConfig iwc = NewIndexWriterConfig(Random, TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random, dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
@ref.CopyChars(TestUtil.RandomUnicodeString(Random));
iw.AddDocument(doc);
if (Random.Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.GetReader();
iw.ForceMerge(1);
DirectoryReader ir2 = iw.GetReader();
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
BinaryDocValues multi = MultiDocValues.GetBinaryValues(ir, "bytes");
BinaryDocValues single = merged.GetBinaryDocValues("bytes");
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
for (int i = 0; i < numDocs; i++)
{
single.Get(i, expected);
multi.Get(i, actual);
Assert.AreEqual(expected, actual);
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}
public static void CheckNorms(AtomicReader reader)
{
// test omit norms
for (int i = 0; i < DocHelper.Fields.Length; i++)
{
IIndexableField f = DocHelper.Fields[i];
if (f.IndexableFieldType.IsIndexed)
{
Assert.AreEqual(reader.GetNormValues(f.Name) != null, !f.IndexableFieldType.OmitNorms);
Assert.AreEqual(reader.GetNormValues(f.Name) != null, !DocHelper.NoNorms.ContainsKey(f.Name));
if (reader.GetNormValues(f.Name) == null)
{
// test for norms of null
NumericDocValues norms = MultiDocValues.GetNormValues(reader, f.Name);
Assert.IsNull(norms);
}
}
}
}
public virtual void TestCustomEncoder()
{
Directory dir = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(Random());
IndexWriterConfig config = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
config.SetSimilarity(new CustomNormEncodingSimilarity(this));
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, config);
Document doc = new Document();
Field foo = NewTextField("foo", "", Field.Store.NO);
Field bar = NewTextField("bar", "", Field.Store.NO);
doc.Add(foo);
doc.Add(bar);
for (int i = 0; i < 100; i++)
{
bar.SetStringValue("singleton");
writer.AddDocument(doc);
}
IndexReader reader = writer.Reader;
writer.Dispose();
NumericDocValues fooNorms = MultiDocValues.GetNormValues(reader, "foo");
for (int i = 0; i < reader.MaxDoc; i++)
{
Assert.AreEqual(0, fooNorms.Get(i));
}
NumericDocValues barNorms = MultiDocValues.GetNormValues(reader, "bar");
for (int i = 0; i < reader.MaxDoc; i++)
{
Assert.AreEqual(1, barNorms.Get(i));
}
reader.Dispose();
dir.Dispose();
}
public virtual void TestNumerics()
{
Directory dir = NewDirectory();
Document doc = new Document();
Field field = new NumericDocValuesField("numbers", 0);
doc.Add(field);
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
field.SetInt64Value(Random().NextLong());
iw.AddDocument(doc);
if (Random().Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.Reader;
iw.ForceMerge(1);
DirectoryReader ir2 = iw.Reader;
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
NumericDocValues multi = MultiDocValues.GetNumericValues(ir, "numbers");
NumericDocValues single = merged.GetNumericDocValues("numbers");
for (int i = 0; i < numDocs; i++)
{
Assert.AreEqual(single.Get(i), multi.Get(i));
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}
public override SortedDocValues GetSortedDocValues(string field)
{
EnsureOpen();
OrdinalMap map = null;
lock (cachedOrdMaps)
{
if (!cachedOrdMaps.TryGetValue(field, out map))
{
// uncached, or not a multi dv
SortedDocValues dv = MultiDocValues.GetSortedValues(@in, field);
MultiSortedDocValues docValues = dv as MultiSortedDocValues;
if (docValues != null)
{
map = docValues.Mapping;
if (map.owner == CoreCacheKey)
{
cachedOrdMaps[field] = map;
}
}
return(dv);
}
}
// cached ordinal map
if (FieldInfos.FieldInfo(field).DocValuesType != DocValuesType.SORTED)
{
return(null);
}
int size = @in.Leaves.Count;
SortedDocValues[] values = new SortedDocValues[size];
int[] starts = new int[size + 1];
for (int i = 0; i < size; i++)
{
AtomicReaderContext context = @in.Leaves[i];
SortedDocValues v = context.AtomicReader.GetSortedDocValues(field) ?? DocValues.EMPTY_SORTED;
values[i] = v;
starts[i] = context.DocBase;
}
starts[size] = MaxDoc;
return(new MultiSortedDocValues(values, starts, map));
}
public virtual void SearchIndex(Directory dir, string oldName)
{
//QueryParser parser = new QueryParser("contents", new MockAnalyzer(random));
//Query query = parser.parse("handle:1");
IndexReader reader = DirectoryReader.Open(dir);
IndexSearcher searcher = NewSearcher(reader);
TestUtil.CheckIndex(dir);
// true if this is a 4.0+ index
bool is40Index = MultiFields.GetMergedFieldInfos(reader).FieldInfo("content5") != null;
// true if this is a 4.2+ index
bool is42Index = MultiFields.GetMergedFieldInfos(reader).FieldInfo("dvSortedSet") != null;
Debug.Assert(is40Index); // NOTE: currently we can only do this on trunk!
Bits liveDocs = MultiFields.GetLiveDocs(reader);
for (int i = 0; i < 35; i++)
{
if (liveDocs.Get(i))
{
Document d = reader.Document(i);
IList <IndexableField> fields = d.Fields;
bool isProxDoc = d.GetField("content3") == null;
if (isProxDoc)
{
int numFields = is40Index ? 7 : 5;
Assert.AreEqual(numFields, fields.Count);
IndexableField f = d.GetField("id");
Assert.AreEqual("" + i, f.StringValue);
f = d.GetField("utf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue);
f = d.GetField("autf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue);
f = d.GetField("content2");
Assert.AreEqual("here is more content with aaa aaa aaa", f.StringValue);
f = d.GetField("fie\u2C77ld");
Assert.AreEqual("field with non-ascii name", f.StringValue);
}
Fields tfvFields = reader.GetTermVectors(i);
Assert.IsNotNull(tfvFields, "i=" + i);
Terms tfv = tfvFields.Terms("utf8");
Assert.IsNotNull(tfv, "docID=" + i + " index=" + oldName);
}
else
{
// Only ID 7 is deleted
Assert.AreEqual(7, i);
}
}
if (is40Index)
{
// check docvalues fields
NumericDocValues dvByte = MultiDocValues.GetNumericValues(reader, "dvByte");
BinaryDocValues dvBytesDerefFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefFixed");
BinaryDocValues dvBytesDerefVar = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefVar");
SortedDocValues dvBytesSortedFixed = MultiDocValues.GetSortedValues(reader, "dvBytesSortedFixed");
SortedDocValues dvBytesSortedVar = MultiDocValues.GetSortedValues(reader, "dvBytesSortedVar");
BinaryDocValues dvBytesStraightFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightFixed");
BinaryDocValues dvBytesStraightVar = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightVar");
NumericDocValues dvDouble = MultiDocValues.GetNumericValues(reader, "dvDouble");
NumericDocValues dvFloat = MultiDocValues.GetNumericValues(reader, "dvFloat");
NumericDocValues dvInt = MultiDocValues.GetNumericValues(reader, "dvInt");
NumericDocValues dvLong = MultiDocValues.GetNumericValues(reader, "dvLong");
NumericDocValues dvPacked = MultiDocValues.GetNumericValues(reader, "dvPacked");
NumericDocValues dvShort = MultiDocValues.GetNumericValues(reader, "dvShort");
SortedSetDocValues dvSortedSet = null;
if (is42Index)
{
dvSortedSet = MultiDocValues.GetSortedSetValues(reader, "dvSortedSet");
}
for (int i = 0; i < 35; i++)
{
int id = Convert.ToInt32(reader.Document(i).Get("id"));
Assert.AreEqual(id, dvByte.Get(i));
sbyte[] bytes = new sbyte[] { (sbyte)((int)((uint)id >> 24)), (sbyte)((int)((uint)id >> 16)), (sbyte)((int)((uint)id >> 8)), (sbyte)id };
BytesRef expectedRef = new BytesRef((byte[])(Array)bytes);
BytesRef scratch = new BytesRef();
dvBytesDerefFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesDerefVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesSortedFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesSortedVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesStraightFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesStraightVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
Assert.AreEqual((double)id, BitConverter.Int64BitsToDouble(dvDouble.Get(i)), 0D);
Assert.AreEqual((float)id, Number.IntBitsToFloat((int)dvFloat.Get(i)), 0F);
Assert.AreEqual(id, dvInt.Get(i));
Assert.AreEqual(id, dvLong.Get(i));
Assert.AreEqual(id, dvPacked.Get(i));
Assert.AreEqual(id, dvShort.Get(i));
if (is42Index)
{
dvSortedSet.Document = i;
long ord = dvSortedSet.NextOrd();
Assert.AreEqual(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.NextOrd());
dvSortedSet.LookupOrd(ord, scratch);
Assert.AreEqual(expectedRef, scratch);
}
}
}
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
// First document should be #0
Document doc = searcher.IndexReader.Document(hits[0].Doc);
assertEquals("didn't get the right document first", "0", doc.Get("id"));
DoTestHits(hits, 34, searcher.IndexReader);
if (is40Index)
{
hits = searcher.Search(new TermQuery(new Term("content5", "aaa")), null, 1000).ScoreDocs;
DoTestHits(hits, 34, searcher.IndexReader);
hits = searcher.Search(new TermQuery(new Term("content6", "aaa")), null, 1000).ScoreDocs;
DoTestHits(hits, 34, searcher.IndexReader);
}
hits = searcher.Search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
reader.Dispose();
}
public virtual void TestDocsWithField()
{
AssumeTrue("codec does not support docsWithField", DefaultCodecSupportsDocsWithField());
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
if (Random().Next(4) >= 0)
{
doc.Add(new NumericDocValuesField("numbers", Random().NextLong()));
}
doc.Add(new NumericDocValuesField("numbersAlways", Random().NextLong()));
iw.AddDocument(doc);
if (Random().Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.Reader;
iw.ForceMerge(1);
DirectoryReader ir2 = iw.Reader;
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
IBits multi = MultiDocValues.GetDocsWithField(ir, "numbers");
IBits single = merged.GetDocsWithField("numbers");
if (multi == null)
{
Assert.IsNull(single);
}
else
{
Assert.AreEqual(single.Length, multi.Length);
for (int i = 0; i < numDocs; i++)
{
Assert.AreEqual(single.Get(i), multi.Get(i));
}
}
multi = MultiDocValues.GetDocsWithField(ir, "numbersAlways");
single = merged.GetDocsWithField("numbersAlways");
Assert.AreEqual(single.Length, multi.Length);
for (int i = 0; i < numDocs; i++)
{
Assert.AreEqual(single.Get(i), multi.Get(i));
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}
public virtual void TestSortedSetWithDups()
{
AssumeTrue("codec does not support SORTED_SET", DefaultCodecSupportsSortedSet());
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
int numValues = Random().Next(5);
for (int j = 0; j < numValues; j++)
{
doc.Add(new SortedSetDocValuesField("bytes", new BytesRef(TestUtil.RandomSimpleString(Random(), 2))));
}
iw.AddDocument(doc);
if (Random().Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.Reader;
iw.ForceMerge(1);
DirectoryReader ir2 = iw.Reader;
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
SortedSetDocValues multi = MultiDocValues.GetSortedSetValues(ir, "bytes");
SortedSetDocValues single = merged.GetSortedSetDocValues("bytes");
if (multi == null)
{
Assert.IsNull(single);
}
else
{
Assert.AreEqual(single.ValueCount, multi.ValueCount);
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
// check values
for (long i = 0; i < single.ValueCount; i++)
{
single.LookupOrd(i, expected);
multi.LookupOrd(i, actual);
Assert.AreEqual(expected, actual);
}
// check ord list
for (int i = 0; i < numDocs; i++)
{
single.SetDocument(i);
List <long?> expectedList = new List <long?>();
long ord;
while ((ord = single.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
expectedList.Add(ord);
}
multi.SetDocument(i);
int upto = 0;
while ((ord = multi.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
Assert.AreEqual((long)expectedList[upto], ord);
upto++;
}
Assert.AreEqual(expectedList.Count, upto);
}
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}
public override IBits GetDocsWithField(string field)
{
EnsureOpen();
return(MultiDocValues.GetDocsWithField(@in, field));
}
public override NumericDocValues GetNumericDocValues(string field)
{
EnsureOpen();
return(MultiDocValues.GetNumericValues(@in, field));
}
public override BinaryDocValues GetBinaryDocValues(string field)
{
EnsureOpen();
return(MultiDocValues.GetBinaryValues(@in, field));
}