/// <summary>
/// Adds a new <see cref="BinaryDocValuesField"/>.
/// </summary>
/// <param name="document">This <see cref="Document"/>.</param>
/// <param name="name"> field name </param>
/// <param name="value"> binary content </param>
/// <returns>The field that was added to this <see cref="Document"/>.</returns>
/// <exception cref="System.ArgumentNullException"> if the field name is null </exception>
public static BinaryDocValuesField AddBinaryDocValuesField(this Document document, string name, BytesRef value)
{
var field = new BinaryDocValuesField(name, value);
document.Add(field);
return(field);
}
private void AddField(Document doc, string field, string value, DocValuesType type)
{
doc.Add(new StringField(field, value, Field.Store.YES));
if (type == DocValuesType.NONE)
{
return;
}
string dvField = field + "_dv";
Field valuesField = null;
switch (type)
{
case DocValuesType.NUMERIC:
valuesField = new NumericDocValuesField(dvField, int.Parse(value, CultureInfo.InvariantCulture));
break;
case DocValuesType.BINARY:
valuesField = new BinaryDocValuesField(dvField, new BytesRef(value));
break;
case DocValuesType.SORTED:
valuesField = new SortedDocValuesField(dvField, new BytesRef(value));
break;
}
doc.Add(valuesField);
}
public virtual void TestFixedBinary()
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BFixedBinary"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetRAMBufferSizeMB(256.0).SetMergeScheduler(new ConcurrentMergeScheduler()).SetMergePolicy(NewLogMergePolicy(false, 10)).SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE));
Document doc = new Document();
sbyte[] bytes = new sbyte[4];
BytesRef data = new BytesRef(bytes);
BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
doc.Add(dvField);
for (int i = 0; i < int.MaxValue; i++)
{
bytes[0] = (sbyte)(i >> 24);
bytes[1] = (sbyte)(i >> 16);
bytes[2] = (sbyte)(i >> 8);
bytes[3] = (sbyte)i;
w.AddDocument(doc);
if (i % 100000 == 0)
{
Console.WriteLine("indexed: " + i);
Console.Out.Flush();
}
}
w.ForceMerge(1);
w.Dispose();
Console.WriteLine("verifying...");
Console.Out.Flush();
DirectoryReader r = DirectoryReader.Open(dir);
int expectedValue = 0;
foreach (AtomicReaderContext context in r.Leaves())
{
AtomicReader reader = (AtomicReader)context.Reader();
BytesRef scratch = new BytesRef();
BinaryDocValues dv = reader.GetBinaryDocValues("dv");
for (int i = 0; i < reader.MaxDoc(); i++)
{
bytes[0] = (sbyte)(expectedValue >> 24);
bytes[1] = (sbyte)(expectedValue >> 16);
bytes[2] = (sbyte)(expectedValue >> 8);
bytes[3] = (sbyte)expectedValue;
dv.Get(i, scratch);
Assert.AreEqual(data, scratch);
expectedValue++;
}
}
r.Dispose();
dir.Dispose();
}
public void TestAddBinaryDocValuesField()
{
BinaryDocValuesField field = null;
BytesRef value = new BytesRef("Foobar");
AssertDocumentExtensionAddsToDocument(document => field = document.AddBinaryDocValuesField("theName", value));
Assert.AreEqual("theName", field.Name);
Assert.AreSame(value, field.FieldsData);
}
/// <summary>
/// Adds a new <see cref="BinaryDocValuesField"/>.
/// </summary>
/// <param name="document">This <see cref="Document"/>.</param>
/// <param name="name"> field name </param>
/// <param name="value"> binary content </param>
/// <returns>The field that was added to this <see cref="Document"/>.</returns>
/// <exception cref="ArgumentNullException"> if this <paramref name="document"/> or the field <paramref name="name"/> is <c>null</c>. </exception>
public static BinaryDocValuesField AddBinaryDocValuesField(this Document document, string name, BytesRef value)
{
if (document is null)
{
throw new ArgumentNullException(nameof(document));
}
var field = new BinaryDocValuesField(name, value);
document.Add(field);
return(field);
}
public virtual void TestBinary()
{
Directory dir = NewDirectory();
Document doc = new Document();
BytesRef @ref = new BytesRef();
Field field = new BinaryDocValuesField("bytes", @ref);
doc.Add(field);
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
iwc.SetMergePolicy(NewLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
@ref.CopyChars(TestUtil.RandomUnicodeString(Random()));
iw.AddDocument(doc);
if (Random().Next(17) == 0)
{
iw.Commit();
}
}
DirectoryReader ir = iw.Reader;
iw.ForceMerge(1);
DirectoryReader ir2 = iw.Reader;
AtomicReader merged = GetOnlySegmentReader(ir2);
iw.Dispose();
BinaryDocValues multi = MultiDocValues.GetBinaryValues(ir, "bytes");
BinaryDocValues single = merged.GetBinaryDocValues("bytes");
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
for (int i = 0; i < numDocs; i++)
{
single.Get(i, expected);
multi.Get(i, actual);
Assert.AreEqual(expected, actual);
}
ir.Dispose();
ir2.Dispose();
dir.Dispose();
}
public virtual void TestBinaryDocValuesField()
{
BinaryDocValuesField field = new BinaryDocValuesField("foo", new BytesRef("bar"));
TrySetBoost(field);
TrySetByteValue(field);
field.BytesValue = "fubar".ToBytesRefArray(Encoding.UTF8);
field.BytesValue = new BytesRef("baz");
TrySetDoubleValue(field);
TrySetIntValue(field);
TrySetFloatValue(field);
TrySetLongValue(field);
TrySetReaderValue(field);
TrySetShortValue(field);
TrySetStringValue(field);
TrySetTokenStreamValue(field);
Assert.AreEqual(new BytesRef("baz"), field.BinaryValue());
}
public virtual void TestBinaryDocValuesField()
{
BinaryDocValuesField field = new BinaryDocValuesField("foo", new BytesRef("bar"));
TrySetBoost(field);
TrySetByteValue(field);
field.BytesValue = "fubar".ToBytesRefArray(Encoding.UTF8);
field.BytesValue = new BytesRef("baz");
TrySetDoubleValue(field);
TrySetIntValue(field);
TrySetFloatValue(field);
TrySetLongValue(field);
TrySetReaderValue(field);
TrySetShortValue(field);
TrySetStringValue(field);
TrySetTokenStreamValue(field);
Assert.AreEqual(new BytesRef("baz"), field.BinaryValue());
}
public virtual void TestMixedTypesDifferentThreads()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
CountdownEvent startingGun = new CountdownEvent(1);
AtomicBoolean hitExc = new AtomicBoolean();
ThreadJob[] threads = new ThreadJob[3];
for (int i = 0; i < 3; i++)
{
Field field;
if (i == 0)
{
field = new SortedDocValuesField("foo", new BytesRef("hello"));
}
else if (i == 1)
{
field = new NumericDocValuesField("foo", 0);
}
else
{
field = new BinaryDocValuesField("foo", new BytesRef("bazz"));
}
Document doc = new Document();
doc.Add(field);
threads[i] = new ThreadAnonymousClass(this, w, startingGun, hitExc, doc);
threads[i].Start();
}
startingGun.Signal();
foreach (ThreadJob t in threads)
{
t.Join();
}
Assert.IsTrue(hitExc);
w.Dispose();
dir.Dispose();
}
public void TestGetField()
{
var target = new BinaryDocValuesField("theName", new BytesRef("Foobar"));
Document document = new Document
{
new BinaryDocValuesField("someOtherName", new BytesRef("Foobar2")),
target
};
BinaryDocValuesField field = document.GetField <BinaryDocValuesField>("theName");
Assert.AreSame(target, field);
Assert.IsNull(document.GetField <BinaryDocValuesField>("nonExistantName"));
#pragma warning disable CS0618 // Type or member is obsolete
Assert.Throws <InvalidCastException>(() => document.GetField <Int32DocValuesField>("theName"));
#pragma warning restore CS0618 // Type or member is obsolete
document = null;
Assert.Throws <ArgumentNullException>(() => document.GetField <BinaryDocValuesField>("theName"));
}
private void AddGroupField(Document doc, string groupField, string value, bool canUseIDV, DocValuesType valueType)
{
doc.Add(new TextField(groupField, value, Field.Store.YES));
if (canUseIDV)
{
Field valuesField = null;
switch (valueType)
{
case DocValuesType.BINARY:
valuesField = new BinaryDocValuesField(groupField + "_dv", new BytesRef(value));
break;
case DocValuesType.SORTED:
valuesField = new SortedDocValuesField(groupField + "_dv", new BytesRef(value));
break;
default:
fail("unhandled type");
break;
}
doc.Add(valuesField);
}
}
public void TestRandom()
{
int numberOfRuns = TestUtil.NextInt32(Random, 3, 6);
for (int iter = 0; iter < numberOfRuns; iter++)
{
if (VERBOSE)
{
Console.WriteLine(string.Format("TEST: iter={0} total={1}", iter, numberOfRuns));
}
int numDocs = TestUtil.NextInt32(Random, 100, 1000) * RANDOM_MULTIPLIER;
int numGroups = TestUtil.NextInt32(Random, 1, numDocs);
if (VERBOSE)
{
Console.WriteLine("TEST: numDocs=" + numDocs + " numGroups=" + numGroups);
}
List <BytesRef> groups = new List <BytesRef>();
for (int i = 0; i < numGroups; i++)
{
string randomValue;
do
{
// B/c of DV based impl we can't see the difference between an empty string and a null value.
// For that reason we don't generate empty string groups.
randomValue = TestUtil.RandomRealisticUnicodeString(Random);
} while ("".Equals(randomValue, StringComparison.Ordinal));
groups.Add(new BytesRef(randomValue));
}
string[] contentStrings = new string[TestUtil.NextInt32(Random, 2, 20)];
if (VERBOSE)
{
Console.WriteLine("TEST: create fake content");
}
for (int contentIDX = 0; contentIDX < contentStrings.Length; contentIDX++)
{
StringBuilder sb = new StringBuilder();
sb.append("real").append(Random.nextInt(3)).append(' ');
int fakeCount = Random.nextInt(10);
for (int fakeIDX = 0; fakeIDX < fakeCount; fakeIDX++)
{
sb.append("fake ");
}
contentStrings[contentIDX] = sb.toString();
if (VERBOSE)
{
Console.WriteLine(" content=" + sb.toString());
}
}
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
Random,
dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(Random)));
bool preFlex = "Lucene3x".Equals(w.IndexWriter.Config.Codec.Name, StringComparison.Ordinal);
bool canUseIDV = !preFlex;
DocValuesType valueType = vts[Random.nextInt(vts.Length)];
Document doc = new Document();
Document docNoGroup = new Document();
Field group = NewStringField("group", "", Field.Store.NO);
doc.Add(group);
Field valuesField = null;
if (canUseIDV)
{
switch (valueType)
{
case DocValuesType.BINARY:
valuesField = new BinaryDocValuesField("group_dv", new BytesRef());
break;
case DocValuesType.SORTED:
valuesField = new SortedDocValuesField("group_dv", new BytesRef());
break;
default:
fail("unhandled type");
break;
}
doc.Add(valuesField);
}
Field sort1 = NewStringField("sort1", "", Field.Store.NO);
doc.Add(sort1);
docNoGroup.Add(sort1);
Field sort2 = NewStringField("sort2", "", Field.Store.NO);
doc.Add(sort2);
docNoGroup.Add(sort2);
Field sort3 = NewStringField("sort3", "", Field.Store.NO);
doc.Add(sort3);
docNoGroup.Add(sort3);
Field content = NewTextField("content", "", Field.Store.NO);
doc.Add(content);
docNoGroup.Add(content);
Int32Field id = new Int32Field("id", 0, Field.Store.NO);
doc.Add(id);
docNoGroup.Add(id);
GroupDoc[] groupDocs = new GroupDoc[numDocs];
for (int i = 0; i < numDocs; i++)
{
BytesRef groupValue;
if (Random.nextInt(24) == 17)
{
// So we test the "doc doesn't have the group'd
// field" case:
groupValue = null;
}
else
{
groupValue = groups[Random.nextInt(groups.size())];
}
GroupDoc groupDoc = new GroupDoc(
i,
groupValue,
groups[Random.nextInt(groups.size())],
groups[Random.nextInt(groups.size())],
new BytesRef(string.Format(CultureInfo.InvariantCulture, "{0:D5}", i)),
contentStrings[Random.nextInt(contentStrings.Length)]
);
if (VERBOSE)
{
Console.WriteLine(" doc content=" + groupDoc.content + " id=" + i + " group=" + (groupDoc.group == null ? "null" : groupDoc.group.Utf8ToString()) + " sort1=" + groupDoc.sort1.Utf8ToString() + " sort2=" + groupDoc.sort2.Utf8ToString() + " sort3=" + groupDoc.sort3.Utf8ToString());
}
groupDocs[i] = groupDoc;
if (groupDoc.group != null)
{
group.SetStringValue(groupDoc.group.Utf8ToString());
if (canUseIDV)
{
valuesField.SetBytesValue(new BytesRef(groupDoc.group.Utf8ToString()));
}
}
sort1.SetStringValue(groupDoc.sort1.Utf8ToString());
sort2.SetStringValue(groupDoc.sort2.Utf8ToString());
sort3.SetStringValue(groupDoc.sort3.Utf8ToString());
content.SetStringValue(groupDoc.content);
id.SetInt32Value(groupDoc.id);
if (groupDoc.group == null)
{
w.AddDocument(docNoGroup);
}
else
{
w.AddDocument(doc);
}
}
DirectoryReader r = w.GetReader();
w.Dispose();
// NOTE: intentional but temporary field cache insanity!
FieldCache.Int32s docIdToFieldId = FieldCache.DEFAULT.GetInt32s(SlowCompositeReaderWrapper.Wrap(r), "id", false);
int[] fieldIdToDocID = new int[numDocs];
for (int i = 0; i < numDocs; i++)
{
int fieldId = docIdToFieldId.Get(i);
fieldIdToDocID[fieldId] = i;
}
try
{
IndexSearcher s = NewSearcher(r);
if (typeof(SlowCompositeReaderWrapper).GetTypeInfo().IsAssignableFrom(s.IndexReader.GetType()))
{
canUseIDV = false;
}
else
{
canUseIDV = !preFlex;
}
for (int contentID = 0; contentID < 3; contentID++)
{
ScoreDoc[] hits = s.Search(new TermQuery(new Term("content", "real" + contentID)), numDocs).ScoreDocs;
foreach (ScoreDoc hit in hits)
{
GroupDoc gd = groupDocs[docIdToFieldId.Get(hit.Doc)];
assertTrue(gd.score == 0.0);
gd.score = hit.Score;
int docId = gd.id;
assertEquals(docId, docIdToFieldId.Get(hit.Doc));
}
}
foreach (GroupDoc gd in groupDocs)
{
assertTrue(gd.score != 0.0);
}
for (int searchIter = 0; searchIter < 100; searchIter++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: searchIter=" + searchIter);
}
string searchTerm = "real" + Random.nextInt(3);
bool sortByScoreOnly = Random.nextBoolean();
Sort sortWithinGroup = GetRandomSort(sortByScoreOnly);
AbstractAllGroupHeadsCollector allGroupHeadsCollector = CreateRandomCollector("group", sortWithinGroup, canUseIDV, valueType);
s.Search(new TermQuery(new Term("content", searchTerm)), allGroupHeadsCollector);
int[] expectedGroupHeads = CreateExpectedGroupHeads(searchTerm, groupDocs, sortWithinGroup, sortByScoreOnly, fieldIdToDocID);
int[] actualGroupHeads = allGroupHeadsCollector.RetrieveGroupHeads();
// The actual group heads contains Lucene ids. Need to change them into our id value.
for (int i = 0; i < actualGroupHeads.Length; i++)
{
actualGroupHeads[i] = docIdToFieldId.Get(actualGroupHeads[i]);
}
// Allows us the easily iterate and assert the actual and expected results.
Array.Sort(expectedGroupHeads);
Array.Sort(actualGroupHeads);
if (VERBOSE)
{
Console.WriteLine("Collector: " + allGroupHeadsCollector.GetType().Name);
Console.WriteLine("Sort within group: " + sortWithinGroup);
Console.WriteLine("Num group: " + numGroups);
Console.WriteLine("Num doc: " + numDocs);
Console.WriteLine("\n=== Expected: \n");
foreach (int expectedDocId in expectedGroupHeads)
{
GroupDoc expectedGroupDoc = groupDocs[expectedDocId];
string expectedGroup = expectedGroupDoc.group == null ? null : expectedGroupDoc.group.Utf8ToString();
Console.WriteLine(
string.Format(CultureInfo.InvariantCulture,
"Group:{0,10} score{1:0.0#######,5} Sort1:{2,10} Sort2:{3,10} Sort3:{4,10} doc:{5,10}",
expectedGroup, expectedGroupDoc.score, expectedGroupDoc.sort1.Utf8ToString(),
expectedGroupDoc.sort2.Utf8ToString(), expectedGroupDoc.sort3.Utf8ToString(), expectedDocId)
);
}
Console.WriteLine("\n=== Actual: \n");
foreach (int actualDocId in actualGroupHeads)
{
GroupDoc actualGroupDoc = groupDocs[actualDocId];
string actualGroup = actualGroupDoc.group == null ? null : actualGroupDoc.group.Utf8ToString();
Console.WriteLine(
string.Format(CultureInfo.InvariantCulture,
"Group:{0,10} score{1:0.0#######,5} Sort1:{2,10} Sort2:{3,10} Sort3:{4,10} doc:{5,10}",
actualGroup, actualGroupDoc.score, actualGroupDoc.sort1.Utf8ToString(),
actualGroupDoc.sort2.Utf8ToString(), actualGroupDoc.sort3.Utf8ToString(), actualDocId)
);
}
Console.WriteLine("\n===================================================================================");
}
assertArrayEquals(expectedGroupHeads, actualGroupHeads);
}
}
finally
{
QueryUtils.PurgeFieldCache(r);
}
r.Dispose();
dir.Dispose();
}
}
public virtual void TestFixedBinary([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler)
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BFixedBinary"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = Throttling.NEVER;
}
var config = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.SetRAMBufferSizeMB(256.0)
.SetMergeScheduler(newScheduler())
.SetMergePolicy(NewLogMergePolicy(false, 10))
.SetOpenMode(OpenMode.CREATE);
IndexWriter w = new IndexWriter(dir, config);
Document doc = new Document();
var bytes = new byte[4];
BytesRef data = new BytesRef(bytes);
BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
doc.Add(dvField);
for (int i = 0; i < int.MaxValue; i++)
{
bytes[0] = (byte)(i >> 24);
bytes[1] = (byte)(i >> 16);
bytes[2] = (byte)(i >> 8);
bytes[3] = (byte)i;
w.AddDocument(doc);
if (i % 100000 == 0)
{
Console.WriteLine("indexed: " + i);
Console.Out.Flush();
}
}
w.ForceMerge(1);
w.Dispose();
Console.WriteLine("verifying...");
Console.Out.Flush();
DirectoryReader r = DirectoryReader.Open(dir);
int expectedValue = 0;
foreach (AtomicReaderContext context in r.Leaves)
{
AtomicReader reader = context.AtomicReader;
BytesRef scratch = new BytesRef();
BinaryDocValues dv = reader.GetBinaryDocValues("dv");
for (int i = 0; i < reader.MaxDoc; i++)
{
bytes[0] = (byte)(expectedValue >> 24);
bytes[1] = (byte)(expectedValue >> 16);
bytes[2] = (byte)(expectedValue >> 8);
bytes[3] = (byte)expectedValue;
dv.Get(i, scratch);
Assert.AreEqual(data, scratch);
expectedValue++;
}
}
r.Dispose();
dir.Dispose();
}
private void DoTest(DocValuesType type)
{
Directory d = NewDirectory();
IndexWriterConfig iwConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
int nDocs = AtLeast(50);
Field id = new NumericDocValuesField("id", 0);
Field f;
switch (type)
{
case DocValuesType.BINARY:
f = new BinaryDocValuesField("dv", new BytesRef());
break;
case DocValuesType.SORTED:
f = new SortedDocValuesField("dv", new BytesRef());
break;
case DocValuesType.NUMERIC:
f = new NumericDocValuesField("dv", 0);
break;
default:
throw AssertionError.Create();
}
Document document = new Document();
document.Add(id);
document.Add(f);
object[] vals = new object[nDocs];
RandomIndexWriter iw = new RandomIndexWriter(Random, d, iwConfig);
for (int i = 0; i < nDocs; ++i)
{
id.SetInt64Value(i);
switch (type)
{
case DocValuesType.SORTED:
case DocValuesType.BINARY:
do
{
vals[i] = TestUtil.RandomSimpleString(Random, 20);
} while (((string)vals[i]).Length == 0);
f.SetBytesValue(new BytesRef((string)vals[i]));
break;
case DocValuesType.NUMERIC:
int bitsPerValue = RandomInts.RandomInt32Between(Random, 1, 31); // keep it an int
vals[i] = (long)Random.Next((int)PackedInt32s.MaxValue(bitsPerValue));
f.SetInt64Value((long)vals[i]);
break;
}
iw.AddDocument(document);
if (Random.NextBoolean() && i % 10 == 9)
{
iw.Commit();
}
}
iw.Dispose();
DirectoryReader rd = DirectoryReader.Open(d);
foreach (AtomicReaderContext leave in rd.Leaves)
{
FunctionValues ids = (new Int64FieldSource("id")).GetValues(null, leave);
ValueSource vs;
switch (type)
{
case DocValuesType.BINARY:
case DocValuesType.SORTED:
vs = new BytesRefFieldSource("dv");
break;
case DocValuesType.NUMERIC:
vs = new Int64FieldSource("dv");
break;
default:
throw AssertionError.Create();
}
FunctionValues values = vs.GetValues(null, leave);
BytesRef bytes = new BytesRef();
for (int i = 0; i < leave.AtomicReader.MaxDoc; ++i)
{
assertTrue(values.Exists(i));
if (vs is BytesRefFieldSource)
{
assertTrue(values.ObjectVal(i) is string);
}
else if (vs is Int64FieldSource)
{
assertTrue(values.ObjectVal(i) is J2N.Numerics.Int64);
assertTrue(values.BytesVal(i, bytes));
}
else
{
throw AssertionError.Create();
}
object expected = vals[ids.Int32Val(i)];
switch (type)
{
case DocValuesType.SORTED:
values.OrdVal(i); // no exception
assertTrue(values.NumOrd >= 1);
goto case DocValuesType.BINARY;
case DocValuesType.BINARY:
assertEquals(expected, values.ObjectVal(i));
assertEquals(expected, values.StrVal(i));
assertEquals(expected, values.ObjectVal(i));
assertEquals(expected, values.StrVal(i));
assertTrue(values.BytesVal(i, bytes));
assertEquals(new BytesRef((string)expected), bytes);
break;
case DocValuesType.NUMERIC:
assertEquals(Convert.ToInt64(expected, CultureInfo.InvariantCulture), values.Int64Val(i));
break;
}
}
}
rd.Dispose();
d.Dispose();
}
// NOTE: this is likely buggy, and cannot clone fields
// with tokenStreamValues, etc. Use at your own risk!!
// TODO: is there a pre-existing way to do this!!!
public static Document CloneDocument(Document doc1)
{
Document doc2 = new Document();
foreach (IIndexableField f in doc1.Fields)
{
Field field1 = (Field)f;
Field field2;
DocValuesType dvType = field1.FieldType.DocValueType;
NumericType numType = field1.FieldType.NumericType;
if (dvType != DocValuesType.NONE)
{
switch (dvType)
{
case DocValuesType.NUMERIC:
field2 = new NumericDocValuesField(field1.Name, field1.GetInt64Value().Value);
break;
case DocValuesType.BINARY:
field2 = new BinaryDocValuesField(field1.Name, field1.GetBinaryValue());
break;
case DocValuesType.SORTED:
field2 = new SortedDocValuesField(field1.Name, field1.GetBinaryValue());
break;
default:
throw IllegalStateException.Create("unknown Type: " + dvType);
}
}
else if (numType != NumericType.NONE)
{
switch (numType)
{
case NumericType.INT32:
field2 = new Int32Field(field1.Name, field1.GetInt32Value().Value, field1.FieldType);
break;
case NumericType.SINGLE:
field2 = new SingleField(field1.Name, field1.GetInt32Value().Value, field1.FieldType);
break;
case NumericType.INT64:
field2 = new Int64Field(field1.Name, field1.GetInt32Value().Value, field1.FieldType);
break;
case NumericType.DOUBLE:
field2 = new DoubleField(field1.Name, field1.GetInt32Value().Value, field1.FieldType);
break;
default:
throw IllegalStateException.Create("unknown Type: " + numType);
}
}
else
{
field2 = new Field(field1.Name, field1.GetStringValue(), field1.FieldType);
}
doc2.Add(field2);
}
return(doc2);
}
public virtual void TestVariableBinary([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler)
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BVariableBinary"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = Throttling.NEVER;
}
var config = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.SetRAMBufferSizeMB(256.0)
.SetMergeScheduler(newScheduler())
.SetMergePolicy(NewLogMergePolicy(false, 10))
.SetOpenMode(OpenMode.CREATE);
IndexWriter w = new IndexWriter(dir, config);
Document doc = new Document();
var bytes = new byte[4];
ByteArrayDataOutput encoder = new ByteArrayDataOutput(bytes);
BytesRef data = new BytesRef(bytes);
BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
doc.Add(dvField);
for (int i = 0; i < int.MaxValue; i++)
{
encoder.Reset(bytes);
encoder.WriteVInt32(i % 65535); // 1, 2, or 3 bytes
data.Length = encoder.Position;
w.AddDocument(doc);
if (i % 100000 == 0)
{
Console.WriteLine("indexed: " + i);
Console.Out.Flush();
}
}
w.ForceMerge(1);
w.Dispose();
Console.WriteLine("verifying...");
Console.Out.Flush();
DirectoryReader r = DirectoryReader.Open(dir);
int expectedValue = 0;
ByteArrayDataInput input = new ByteArrayDataInput();
foreach (AtomicReaderContext context in r.Leaves)
{
AtomicReader reader = context.AtomicReader;
BytesRef scratch = new BytesRef(bytes);
BinaryDocValues dv = reader.GetBinaryDocValues("dv");
for (int i = 0; i < reader.MaxDoc; i++)
{
dv.Get(i, scratch);
input.Reset(scratch.Bytes, scratch.Offset, scratch.Length);
Assert.AreEqual(expectedValue % 65535, input.ReadVInt32());
Assert.IsTrue(input.Eof);
expectedValue++;
}
}
r.Dispose();
dir.Dispose();
}
public virtual void TestVariableBinary()
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BVariableBinary"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetRAMBufferSizeMB(256.0).SetMergeScheduler(new ConcurrentMergeScheduler()).SetMergePolicy(NewLogMergePolicy(false, 10)).SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE));
Document doc = new Document();
sbyte[] bytes = new sbyte[4];
ByteArrayDataOutput encoder = new ByteArrayDataOutput((byte[])(Array)bytes);
BytesRef data = new BytesRef(bytes);
BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
doc.Add(dvField);
for (int i = 0; i < int.MaxValue; i++)
{
encoder.Reset((byte[])(Array)bytes);
encoder.WriteVInt(i % 65535); // 1, 2, or 3 bytes
data.Length = encoder.Position;
w.AddDocument(doc);
if (i % 100000 == 0)
{
Console.WriteLine("indexed: " + i);
Console.Out.Flush();
}
}
w.ForceMerge(1);
w.Dispose();
Console.WriteLine("verifying...");
Console.Out.Flush();
DirectoryReader r = DirectoryReader.Open(dir);
int expectedValue = 0;
ByteArrayDataInput input = new ByteArrayDataInput();
foreach (AtomicReaderContext context in r.Leaves())
{
AtomicReader reader = (AtomicReader)context.Reader();
BytesRef scratch = new BytesRef(bytes);
BinaryDocValues dv = reader.GetBinaryDocValues("dv");
for (int i = 0; i < reader.MaxDoc(); i++)
{
dv.Get(i, scratch);
input.Reset((byte[])(Array)scratch.Bytes, scratch.Offset, scratch.Length);
Assert.AreEqual(expectedValue % 65535, input.ReadVInt());
Assert.IsTrue(input.Eof());
expectedValue++;
}
}
r.Dispose();
dir.Dispose();
}