public virtual void TestMixing()
{
// no positions
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.IndexOptions = IndexOptions.DOCS_AND_FREQS;
Directory dir = NewDirectory();
RandomIndexWriter iw = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
for (int i = 0; i < 20; i++)
{
Document doc = new Document();
if (i < 19 && Random.NextBoolean())
{
for (int j = 0; j < 50; j++)
{
doc.Add(new TextField("foo", "i have positions", Field.Store.NO));
}
}
else
{
for (int j = 0; j < 50; j++)
{
doc.Add(new Field("foo", "i have no positions", ft));
}
}
iw.AddDocument(doc);
iw.Commit();
}
if (Random.NextBoolean())
{
iw.ForceMerge(1);
}
DirectoryReader ir = iw.GetReader();
FieldInfos fis = MultiFields.GetMergedFieldInfos(ir);
Assert.AreEqual(IndexOptions.DOCS_AND_FREQS, fis.FieldInfo("foo").IndexOptions);
Assert.IsFalse(fis.FieldInfo("foo").HasPayloads);
iw.Dispose();
ir.Dispose();
dir.Dispose(); // checkindex
}
/// <summary>
/// Returns a <see cref="NumericDocValues"/> for a reader's norms (potentially merging on-the-fly).
/// <para>
/// This is a slow way to access normalization values. Instead, access them per-segment
/// with <seealso cref="AtomicReader.GetNormValues(string)"/>
/// </para>
/// </summary>
public static NumericDocValues GetNormValues(IndexReader r, string field)
{
IList <AtomicReaderContext> leaves = r.Leaves;
int size = leaves.Count;
if (size == 0)
{
return(null);
}
else if (size == 1)
{
return(leaves[0].AtomicReader.GetNormValues(field));
}
FieldInfo fi = MultiFields.GetMergedFieldInfos(r).FieldInfo(field);
if (fi == null || fi.HasNorms == false)
{
return(null);
}
bool anyReal = false;
NumericDocValues[] values = new NumericDocValues[size];
int[] starts = new int[size + 1];
for (int i = 0; i < size; i++)
{
AtomicReaderContext context = leaves[i];
NumericDocValues v = context.AtomicReader.GetNormValues(field);
if (v == null)
{
v = DocValues.EMPTY_NUMERIC;
}
else
{
anyReal = true;
}
values[i] = v;
starts[i] = context.DocBase;
}
starts[size] = r.MaxDoc;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(anyReal);
}
return(new NumericDocValuesAnonymousInnerClassHelper(values, starts));
}
public virtual void TestMixing()
{
// no positions
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS;
Directory dir = NewDirectory();
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
for (int i = 0; i < 20; i++)
{
Document doc = new Document();
if (i < 19 && Random().NextBoolean())
{
for (int j = 0; j < 50; j++)
{
doc.Add(new TextField("foo", "i have positions", Field.Store.NO));
}
}
else
{
for (int j = 0; j < 50; j++)
{
doc.Add(new Field("foo", "i have no positions", ft));
}
}
iw.AddDocument(doc);
iw.Commit();
}
if (Random().NextBoolean())
{
iw.ForceMerge(1);
}
DirectoryReader ir = iw.Reader;
FieldInfos fis = MultiFields.GetMergedFieldInfos(ir);
Assert.AreEqual(FieldInfo.IndexOptions.DOCS_AND_FREQS, fis.FieldInfo("foo").FieldIndexOptions);
Assert.IsFalse(fis.FieldInfo("foo").HasPayloads());
iw.Dispose();
ir.Dispose();
dir.Dispose(); // checkindex
}
public virtual void SearchIndex(Directory dir, string oldName)
{
//QueryParser parser = new QueryParser("contents", new MockAnalyzer(random));
//Query query = parser.parse("handle:1");
IndexReader reader = DirectoryReader.Open(dir);
IndexSearcher searcher = NewSearcher(reader);
TestUtil.CheckIndex(dir);
// true if this is a 4.0+ index
bool is40Index = MultiFields.GetMergedFieldInfos(reader).FieldInfo("content5") != null;
// true if this is a 4.2+ index
bool is42Index = MultiFields.GetMergedFieldInfos(reader).FieldInfo("dvSortedSet") != null;
Debug.Assert(is40Index); // NOTE: currently we can only do this on trunk!
Bits liveDocs = MultiFields.GetLiveDocs(reader);
for (int i = 0; i < 35; i++)
{
if (liveDocs.Get(i))
{
Document d = reader.Document(i);
IList <IndexableField> fields = d.Fields;
bool isProxDoc = d.GetField("content3") == null;
if (isProxDoc)
{
int numFields = is40Index ? 7 : 5;
Assert.AreEqual(numFields, fields.Count);
IndexableField f = d.GetField("id");
Assert.AreEqual("" + i, f.StringValue);
f = d.GetField("utf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue);
f = d.GetField("autf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue);
f = d.GetField("content2");
Assert.AreEqual("here is more content with aaa aaa aaa", f.StringValue);
f = d.GetField("fie\u2C77ld");
Assert.AreEqual("field with non-ascii name", f.StringValue);
}
Fields tfvFields = reader.GetTermVectors(i);
Assert.IsNotNull(tfvFields, "i=" + i);
Terms tfv = tfvFields.Terms("utf8");
Assert.IsNotNull(tfv, "docID=" + i + " index=" + oldName);
}
else
{
// Only ID 7 is deleted
Assert.AreEqual(7, i);
}
}
if (is40Index)
{
// check docvalues fields
NumericDocValues dvByte = MultiDocValues.GetNumericValues(reader, "dvByte");
BinaryDocValues dvBytesDerefFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefFixed");
BinaryDocValues dvBytesDerefVar = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefVar");
SortedDocValues dvBytesSortedFixed = MultiDocValues.GetSortedValues(reader, "dvBytesSortedFixed");
SortedDocValues dvBytesSortedVar = MultiDocValues.GetSortedValues(reader, "dvBytesSortedVar");
BinaryDocValues dvBytesStraightFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightFixed");
BinaryDocValues dvBytesStraightVar = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightVar");
NumericDocValues dvDouble = MultiDocValues.GetNumericValues(reader, "dvDouble");
NumericDocValues dvFloat = MultiDocValues.GetNumericValues(reader, "dvFloat");
NumericDocValues dvInt = MultiDocValues.GetNumericValues(reader, "dvInt");
NumericDocValues dvLong = MultiDocValues.GetNumericValues(reader, "dvLong");
NumericDocValues dvPacked = MultiDocValues.GetNumericValues(reader, "dvPacked");
NumericDocValues dvShort = MultiDocValues.GetNumericValues(reader, "dvShort");
SortedSetDocValues dvSortedSet = null;
if (is42Index)
{
dvSortedSet = MultiDocValues.GetSortedSetValues(reader, "dvSortedSet");
}
for (int i = 0; i < 35; i++)
{
int id = Convert.ToInt32(reader.Document(i).Get("id"));
Assert.AreEqual(id, dvByte.Get(i));
sbyte[] bytes = new sbyte[] { (sbyte)((int)((uint)id >> 24)), (sbyte)((int)((uint)id >> 16)), (sbyte)((int)((uint)id >> 8)), (sbyte)id };
BytesRef expectedRef = new BytesRef((byte[])(Array)bytes);
BytesRef scratch = new BytesRef();
dvBytesDerefFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesDerefVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesSortedFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesSortedVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesStraightFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesStraightVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
Assert.AreEqual((double)id, BitConverter.Int64BitsToDouble(dvDouble.Get(i)), 0D);
Assert.AreEqual((float)id, Number.IntBitsToFloat((int)dvFloat.Get(i)), 0F);
Assert.AreEqual(id, dvInt.Get(i));
Assert.AreEqual(id, dvLong.Get(i));
Assert.AreEqual(id, dvPacked.Get(i));
Assert.AreEqual(id, dvShort.Get(i));
if (is42Index)
{
dvSortedSet.Document = i;
long ord = dvSortedSet.NextOrd();
Assert.AreEqual(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.NextOrd());
dvSortedSet.LookupOrd(ord, scratch);
Assert.AreEqual(expectedRef, scratch);
}
}
}
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
// First document should be #0
Document doc = searcher.IndexReader.Document(hits[0].Doc);
assertEquals("didn't get the right document first", "0", doc.Get("id"));
DoTestHits(hits, 34, searcher.IndexReader);
if (is40Index)
{
hits = searcher.Search(new TermQuery(new Term("content5", "aaa")), null, 1000).ScoreDocs;
DoTestHits(hits, 34, searcher.IndexReader);
hits = searcher.Search(new TermQuery(new Term("content6", "aaa")), null, 1000).ScoreDocs;
DoTestHits(hits, 34, searcher.IndexReader);
}
hits = searcher.Search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
reader.Dispose();
}
