| public abstract GetSortedDocValues ( string field ) : Lucene.Net.Index.SortedDocValues | ||
| field | string | |
| return | Lucene.Net.Index.SortedDocValues | |
public override void Run()
{
SortedDocValues stringDVDirect;
NumericDocValues docIDToID;
try
{
stringDVDirect = sr.GetSortedDocValues("stringdv");
docIDToID = sr.GetNumericDocValues("id");
Assert.IsNotNull(stringDVDirect);
}
catch (IOException ioe)
{
throw new Exception(ioe.ToString(), ioe);
}
while (Environment.TickCount < endTime)
{
SortedDocValues source;
source = stringDVDirect;
BytesRef scratch = new BytesRef();
for (int iter = 0; iter < 100; iter++)
{
int docID = random.Next(sr.MaxDoc);
source.Get(docID, scratch);
Assert.AreEqual(docValues[(int)docIDToID.Get(docID)], scratch);
}
}
}
public override void Run()
{
SortedDocValues stringDVDirect;
NumericDocValues docIDToID;
try
{
stringDVDirect = sr.GetSortedDocValues("stringdv");
docIDToID = sr.GetNumericDocValues("id");
Assert.IsNotNull(stringDVDirect);
}
catch (Exception ioe) when(ioe.IsIOException())
{
throw RuntimeException.Create(ioe);
}
while (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond < endTime) // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
{
SortedDocValues source;
source = stringDVDirect;
BytesRef scratch = new BytesRef();
for (int iter = 0; iter < 100; iter++)
{
int docID = random.Next(sr.MaxDoc);
source.Get(docID, scratch);
Assert.AreEqual(docValues[(int)docIDToID.Get(docID)], scratch);
}
}
}
public override void Warm(AtomicReader reader)
{
long startTime = Environment.TickCount;
int indexedCount = 0;
int docValuesCount = 0;
int normsCount = 0;
foreach (FieldInfo info in reader.FieldInfos)
{
if (info.IsIndexed)
{
reader.GetTerms(info.Name);
indexedCount++;
if (info.HasNorms)
{
reader.GetNormValues(info.Name);
normsCount++;
}
}
if (info.HasDocValues)
{
switch (info.DocValuesType)
{
case DocValuesType.NUMERIC:
reader.GetNumericDocValues(info.Name);
break;
case DocValuesType.BINARY:
reader.GetBinaryDocValues(info.Name);
break;
case DocValuesType.SORTED:
reader.GetSortedDocValues(info.Name);
break;
case DocValuesType.SORTED_SET:
reader.GetSortedSetDocValues(info.Name);
break;
default:
if (Debugging.AssertsEnabled)
{
Debugging.Assert(false); // unknown dv type
}
break;
}
docValuesCount++;
}
}
reader.Document(0);
reader.GetTermVectors(0);
if (infoStream.IsEnabled("SMSW"))
{
infoStream.Message("SMSW", "Finished warming segment: " + reader + ", indexed=" + indexedCount + ", docValues=" + docValuesCount + ", norms=" + normsCount + ", time=" + (Environment.TickCount - startTime));
}
}
public override void Run()
{
Random random = Random();
SortedDocValues stringDVDirect;
NumericDocValues docIDToID;
try
{
stringDVDirect = Sr.GetSortedDocValues("stringdv");
docIDToID = Sr.GetNumericDocValues("id");
Assert.IsNotNull(stringDVDirect);
}
catch (IOException ioe)
{
throw new Exception(ioe.Message, ioe);
}
while (DateTime.Now.Millisecond < END_TIME)
{
SortedDocValues source;
source = stringDVDirect;
BytesRef scratch = new BytesRef();
for (int iter = 0; iter < 100; iter++)
{
int docID = random.Next(Sr.MaxDoc);
source.Get(docID, scratch);
Assert.AreEqual(DocValues[(int)docIDToID.Get(docID)], scratch);
}
}
}
public virtual void TestFixedSorted()
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BFixedSorted"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetRAMBufferSizeMB(256.0).SetMergeScheduler(new ConcurrentMergeScheduler()).SetMergePolicy(NewLogMergePolicy(false, 10)).SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE));
Document doc = new Document();
sbyte[] bytes = new sbyte[2];
BytesRef data = new BytesRef(bytes);
SortedDocValuesField dvField = new SortedDocValuesField("dv", data);
doc.Add(dvField);
for (int i = 0; i < int.MaxValue; i++)
{
bytes[0] = (sbyte)(i >> 8);
bytes[1] = (sbyte)i;
w.AddDocument(doc);
if (i % 100000 == 0)
{
Console.WriteLine("indexed: " + i);
Console.Out.Flush();
}
}
w.ForceMerge(1);
w.Dispose();
Console.WriteLine("verifying...");
Console.Out.Flush();
DirectoryReader r = DirectoryReader.Open(dir);
int expectedValue = 0;
foreach (AtomicReaderContext context in r.Leaves)
{
AtomicReader reader = context.AtomicReader;
BytesRef scratch = new BytesRef();
BinaryDocValues dv = reader.GetSortedDocValues("dv");
for (int i = 0; i < reader.MaxDoc; i++)
{
bytes[0] = (sbyte)(expectedValue >> 8);
bytes[1] = (sbyte)expectedValue;
dv.Get(i, scratch);
Assert.AreEqual(data, scratch);
expectedValue++;
}
}
r.Dispose();
dir.Dispose();
}
public virtual void TestSorted()
{
Directory dir = NewDirectory();
Document doc = new Document();
BytesRef @ref = new BytesRef();
Field field = new SortedDocValuesField("bytes", @ref);
doc.Add(field);
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
@ref.CopyChars(TestUtil.RandomUnicodeString(Random()));
if (DefaultCodecSupportsDocsWithField() && Random().Next(7) == 0)
{
iw.AddDocument(new Document());
}
iw.AddDocument(doc);
if (Random().Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.Reader;
iw.ForceMerge(1);
DirectoryReader ir2 = iw.Reader;
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
SortedDocValues multi = MultiDocValues.GetSortedValues(ir, "bytes");
SortedDocValues single = merged.GetSortedDocValues("bytes");
Assert.AreEqual(single.ValueCount, multi.ValueCount);
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
for (int i = 0; i < numDocs; i++)
{
// check ord
Assert.AreEqual(single.GetOrd(i), multi.GetOrd(i));
// check value
single.Get(i, expected);
multi.Get(i, actual);
Assert.AreEqual(expected, actual);
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}
public override void Warm(AtomicReader reader)
{
long startTime = DateTime.Now.Millisecond;
int indexedCount = 0;
int docValuesCount = 0;
int normsCount = 0;
foreach (FieldInfo info in reader.FieldInfos)
{
if (info.Indexed)
{
reader.Terms(info.Name);
indexedCount++;
if (info.HasNorms())
{
reader.GetNormValues(info.Name);
normsCount++;
}
}
if (info.HasDocValues())
{
switch (info.DocValuesType)
{
case DocValuesType_e.NUMERIC:
reader.GetNumericDocValues(info.Name);
break;
case DocValuesType_e.BINARY:
reader.GetBinaryDocValues(info.Name);
break;
case DocValuesType_e.SORTED:
reader.GetSortedDocValues(info.Name);
break;
case DocValuesType_e.SORTED_SET:
reader.GetSortedSetDocValues(info.Name);
break;
default:
Debug.Assert(false); // unknown dv type
break;
}
docValuesCount++;
}
}
reader.Document(0);
reader.GetTermVectors(0);
if (InfoStream.IsEnabled("SMSW"))
{
InfoStream.Message("SMSW", "Finished warming segment: " + reader + ", indexed=" + indexedCount + ", docValues=" + docValuesCount + ", norms=" + normsCount + ", time=" + (DateTime.Now.Millisecond - startTime));
}
}
/// <summary>
/// Test docvalues.
/// @lucene.experimental
/// </summary>
public static Status.DocValuesStatus TestDocValues(AtomicReader reader, TextWriter infoStream)
{
Status.DocValuesStatus status = new Status.DocValuesStatus();
try
{
if (infoStream != null)
{
infoStream.Write(" test: docvalues...........");
}
foreach (FieldInfo fieldInfo in reader.FieldInfos)
{
if (fieldInfo.HasDocValues())
{
status.TotalValueFields++;
CheckDocValues(fieldInfo, reader, /*infoStream,*/ status);
}
else
{
if (reader.GetBinaryDocValues(fieldInfo.Name) != null || reader.GetNumericDocValues(fieldInfo.Name) != null || reader.GetSortedDocValues(fieldInfo.Name) != null || reader.GetSortedSetDocValues(fieldInfo.Name) != null || reader.GetDocsWithField(fieldInfo.Name) != null)
{
throw new Exception("field: " + fieldInfo.Name + " has docvalues but should omit them!");
}
}
}
Msg(infoStream, "OK [" + status.TotalValueFields + " docvalues fields; " + status.TotalBinaryFields + " BINARY; " + status.TotalNumericFields + " NUMERIC; " + status.TotalSortedFields + " SORTED; " + status.TotalSortedSetFields + " SORTED_SET]");
}
catch (Exception e)
{
Msg(infoStream, "ERROR [" + Convert.ToString(e.Message) + "]");
status.Error = e;
if (infoStream != null)
{
// LUCENENET NOTE: Some tests rely on the error type being in
// the message. We can't get the error type with StackTrace, we
// need ToString() for that.
infoStream.WriteLine(e.ToString());
//infoStream.WriteLine(e.StackTrace);
}
}
return status;
}
private static void CheckDocValues(FieldInfo fi, AtomicReader reader, /*StreamWriter infoStream,*/ DocValuesStatus status)
{
Bits docsWithField = reader.GetDocsWithField(fi.Name);
if (docsWithField == null)
{
throw new Exception(fi.Name + " docsWithField does not exist");
}
else if (docsWithField.Length() != reader.MaxDoc)
{
throw new Exception(fi.Name + " docsWithField has incorrect length: " + docsWithField.Length() + ",expected: " + reader.MaxDoc);
}
switch (fi.DocValuesType)
{
case FieldInfo.DocValuesType_e.SORTED:
status.TotalSortedFields++;
CheckSortedDocValues(fi.Name, reader, reader.GetSortedDocValues(fi.Name), docsWithField);
if (reader.GetBinaryDocValues(fi.Name) != null || reader.GetNumericDocValues(fi.Name) != null || reader.GetSortedSetDocValues(fi.Name) != null)
{
throw new Exception(fi.Name + " returns multiple docvalues types!");
}
break;
case FieldInfo.DocValuesType_e.SORTED_SET:
status.TotalSortedSetFields++;
CheckSortedSetDocValues(fi.Name, reader, reader.GetSortedSetDocValues(fi.Name), docsWithField);
if (reader.GetBinaryDocValues(fi.Name) != null || reader.GetNumericDocValues(fi.Name) != null || reader.GetSortedDocValues(fi.Name) != null)
{
throw new Exception(fi.Name + " returns multiple docvalues types!");
}
break;
case FieldInfo.DocValuesType_e.BINARY:
status.TotalBinaryFields++;
CheckBinaryDocValues(fi.Name, reader, reader.GetBinaryDocValues(fi.Name), docsWithField);
if (reader.GetNumericDocValues(fi.Name) != null || reader.GetSortedDocValues(fi.Name) != null || reader.GetSortedSetDocValues(fi.Name) != null)
{
throw new Exception(fi.Name + " returns multiple docvalues types!");
}
break;
case FieldInfo.DocValuesType_e.NUMERIC:
status.TotalNumericFields++;
CheckNumericDocValues(fi.Name, reader, reader.GetNumericDocValues(fi.Name), docsWithField);
if (reader.GetBinaryDocValues(fi.Name) != null || reader.GetSortedDocValues(fi.Name) != null || reader.GetSortedSetDocValues(fi.Name) != null)
{
throw new Exception(fi.Name + " returns multiple docvalues types!");
}
break;
default:
throw new InvalidOperationException();
}
}
public override void Warm(AtomicReader reader)
{
long startTime = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
int indexedCount = 0;
int docValuesCount = 0;
int normsCount = 0;
foreach (FieldInfo info in reader.FieldInfos)
{
if (info.IsIndexed)
{
reader.GetTerms(info.Name);
indexedCount++;
if (info.HasNorms)
{
reader.GetNormValues(info.Name);
normsCount++;
}
}
if (info.HasDocValues)
{
switch (info.DocValuesType)
{
case DocValuesType.NUMERIC:
reader.GetNumericDocValues(info.Name);
break;
case DocValuesType.BINARY:
reader.GetBinaryDocValues(info.Name);
break;
case DocValuesType.SORTED:
reader.GetSortedDocValues(info.Name);
break;
case DocValuesType.SORTED_SET:
reader.GetSortedSetDocValues(info.Name);
break;
default:
if (Debugging.AssertsEnabled)
{
Debugging.Assert(false); // unknown dv type
}
break;
}
docValuesCount++;
}
}
reader.Document(0);
reader.GetTermVectors(0);
if (infoStream.IsEnabled("SMSW"))
{
infoStream.Message("SMSW",
"Finished warming segment: " + reader +
", indexed=" + indexedCount +
", docValues=" + docValuesCount +
", norms=" + normsCount +
", time=" + ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - startTime)); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
}
}
public override SortedDocValues GetSortedDocValues(string field)
{
EnsureOpen();
return(m_input.GetSortedDocValues(field));
}
public virtual void Test2BOrds([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler)
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BOrds"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = Throttling.NEVER;
}
var config = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.SetRAMBufferSizeMB(256.0)
.SetMergeScheduler(newScheduler())
.SetMergePolicy(NewLogMergePolicy(false, 10))
.SetOpenMode(OpenMode.CREATE);
IndexWriter w = new IndexWriter(dir, config);
Document doc = new Document();
var bytes = new byte[4];
BytesRef data = new BytesRef(bytes);
SortedDocValuesField dvField = new SortedDocValuesField("dv", data);
doc.Add(dvField);
for (int i = 0; i < int.MaxValue; i++)
{
bytes[0] = (byte)(i >> 24);
bytes[1] = (byte)(i >> 16);
bytes[2] = (byte)(i >> 8);
bytes[3] = (byte)i;
w.AddDocument(doc);
if (i % 100000 == 0)
{
Console.WriteLine("indexed: " + i);
Console.Out.Flush();
}
}
w.ForceMerge(1);
w.Dispose();
Console.WriteLine("verifying...");
Console.Out.Flush();
DirectoryReader r = DirectoryReader.Open(dir);
int counter = 0;
foreach (AtomicReaderContext context in r.Leaves)
{
AtomicReader reader = context.AtomicReader;
BytesRef scratch = new BytesRef();
BinaryDocValues dv = reader.GetSortedDocValues("dv");
for (int i = 0; i < reader.MaxDoc; i++)
{
bytes[0] = (byte)(counter >> 24);
bytes[1] = (byte)(counter >> 16);
bytes[2] = (byte)(counter >> 8);
bytes[3] = (byte)counter;
counter++;
dv.Get(i, scratch);
Assert.AreEqual(data, scratch);
}
}
r.Dispose();
dir.Dispose();
}