internal void WriteField(FieldInfo fi, IFieldable field, IState state)
{
fieldsStream.WriteVInt(fi.number);
byte bits = 0;
if (field.IsTokenized)
{
bits |= FieldsWriter.FIELD_IS_TOKENIZED;
}
if (field.IsBinary)
{
bits |= FieldsWriter.FIELD_IS_BINARY;
}
fieldsStream.WriteByte(bits);
// compression is disabled for the current field
if (field.IsBinary)
{
byte[] data = field.GetBinaryValue(state);
int len = field.BinaryLength;
int offset = field.BinaryOffset;
fieldsStream.WriteVInt(len);
fieldsStream.WriteBytes(data, offset, len);
}
else
{
fieldsStream.WriteString(field.StringValue(state));
}
}
public virtual void TestLazyFields()
{
Assert.IsTrue(dir != null);
Assert.IsTrue(fieldInfos != null);
FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos, null);
Assert.IsTrue(reader != null);
Assert.IsTrue(reader.Size() == 1);
ISet <string> loadFieldNames = Support.Compatibility.SetFactory.CreateHashSet <string>();
loadFieldNames.Add(DocHelper.TEXT_FIELD_1_KEY);
loadFieldNames.Add(DocHelper.TEXT_FIELD_UTF1_KEY);
ISet <string> lazyFieldNames = Support.Compatibility.SetFactory.CreateHashSet <string>();
//new String[]{DocHelper.LARGE_LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_BINARY_KEY};
lazyFieldNames.Add(DocHelper.LARGE_LAZY_FIELD_KEY);
lazyFieldNames.Add(DocHelper.LAZY_FIELD_KEY);
lazyFieldNames.Add(DocHelper.LAZY_FIELD_BINARY_KEY);
lazyFieldNames.Add(DocHelper.TEXT_FIELD_UTF2_KEY);
SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames);
Document doc = reader.Doc(0, fieldSelector, null);
Assert.IsTrue(doc != null, "doc is null and it shouldn't be");
IFieldable field = doc.GetFieldable(DocHelper.LAZY_FIELD_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.IsLazy, "field is not lazy and it should be");
System.String value_Renamed = field.StringValue(null);
Assert.IsTrue(value_Renamed != null, "value is null and it shouldn't be");
Assert.IsTrue(value_Renamed.Equals(DocHelper.LAZY_FIELD_TEXT) == true, value_Renamed + " is not equal to " + DocHelper.LAZY_FIELD_TEXT);
field = doc.GetFieldable(DocHelper.TEXT_FIELD_1_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.IsLazy == false, "Field is lazy and it should not be");
field = doc.GetFieldable(DocHelper.TEXT_FIELD_UTF1_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.IsLazy == false, "Field is lazy and it should not be");
Assert.IsTrue(field.StringValue(null).Equals(DocHelper.FIELD_UTF1_TEXT) == true, field.StringValue(null) + " is not equal to " + DocHelper.FIELD_UTF1_TEXT);
field = doc.GetFieldable(DocHelper.TEXT_FIELD_UTF2_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.IsLazy == true, "Field is lazy and it should not be");
Assert.IsTrue(field.StringValue(null).Equals(DocHelper.FIELD_UTF2_TEXT) == true, field.StringValue(null) + " is not equal to " + DocHelper.FIELD_UTF2_TEXT);
field = doc.GetFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
Assert.IsTrue(field != null, "field is null and it shouldn't be");
Assert.IsTrue(field.StringValue(null) == null, "stringValue isn't null for lazy binary field");
byte[] bytes = field.GetBinaryValue(null);
Assert.IsTrue(bytes != null, "bytes is null and it shouldn't be");
Assert.IsTrue(DocHelper.LAZY_FIELD_BINARY_BYTES.Length == bytes.Length, "");
for (int i = 0; i < bytes.Length; i++)
{
Assert.IsTrue(bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i], "byte[" + i + "] is mismatched");
}
}
public virtual void TestLoadSize()
{
FieldsReader reader = new FieldsReader(dir, TEST_SEGMENT_NAME, fieldInfos, null);
Document doc;
doc = reader.Doc(0, new AnonymousClassFieldSelector(this), null);
IFieldable f1 = doc.GetFieldable(DocHelper.TEXT_FIELD_1_KEY);
IFieldable f3 = doc.GetFieldable(DocHelper.TEXT_FIELD_3_KEY);
IFieldable fb = doc.GetFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
Assert.IsTrue(f1.IsBinary);
Assert.IsTrue(!f3.IsBinary);
Assert.IsTrue(fb.IsBinary);
AssertSizeEquals(2 * DocHelper.FIELD_1_TEXT.Length, f1.GetBinaryValue(null));
Assert.AreEqual(DocHelper.FIELD_3_TEXT, f3.StringValue(null));
AssertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.Length, fb.GetBinaryValue(null));
reader.Dispose();
}
private static KeyValuePair <string, JToken> CreateProperty(IFieldable fld, Document document)
{
if (fld.IsBinary)
{
return(new KeyValuePair <string, JToken>(fld.Name, fld.GetBinaryValue()));
}
var stringValue = fld.StringValue;
if (document.GetField(fld.Name + "_ConvertToJson") != null)
{
var val = JToken.Parse(fld.StringValue) as JObject;
return(new KeyValuePair <string, JToken>(fld.Name, val));
}
if (stringValue == Constants.NullValue)
{
stringValue = null;
}
if (stringValue == Constants.EmptyString)
{
stringValue = string.Empty;
}
return(new KeyValuePair <string, JToken>(fld.Name, stringValue));
}
private void assertCompressedFields29(Directory dir, bool shouldStillBeCompressed)
{
int count = 0;
int TEXT_PLAIN_LENGTH = TEXT_TO_COMPRESS.Length * 2;
// FieldSelectorResult.SIZE returns 2*number_of_chars for String fields:
int BINARY_PLAIN_LENGTH = BINARY_TO_COMPRESS.Length;
IndexReader reader = IndexReader.Open(dir, true);
try
{
// look into sub readers and check if raw merge is on/off
var readers = new System.Collections.Generic.List <IndexReader>();
ReaderUtil.GatherSubReaders(readers, reader);
foreach (IndexReader ir in readers)
{
FieldsReader fr = ((SegmentReader)ir).GetFieldsReader();
Assert.IsTrue(shouldStillBeCompressed != fr.CanReadRawDocs(),
"for a 2.9 index, FieldsReader.canReadRawDocs() must be false and other way round for a trunk index");
}
// test that decompression works correctly
for (int i = 0; i < reader.MaxDoc; i++)
{
if (!reader.IsDeleted(i))
{
Document d = reader.Document(i);
if (d.Get("content3") != null)
{
continue;
}
count++;
IFieldable compressed = d.GetFieldable("compressed");
if (int.Parse(d.Get("id")) % 2 == 0)
{
Assert.IsFalse(compressed.IsBinary);
Assert.AreEqual(TEXT_TO_COMPRESS, compressed.StringValue,
"incorrectly decompressed string");
}
else
{
Assert.IsTrue(compressed.IsBinary);
Assert.IsTrue(BINARY_TO_COMPRESS.SequenceEqual(compressed.GetBinaryValue()),
"incorrectly decompressed binary");
}
}
}
//check if field was decompressed after optimize
for (int i = 0; i < reader.MaxDoc; i++)
{
if (!reader.IsDeleted(i))
{
Document d = reader.Document(i, new AnonymousFieldSelector());
if (d.Get("content3") != null)
{
continue;
}
count++;
// read the size from the binary value using BinaryReader (this prevents us from doing the shift ops ourselves):
// ugh, Java uses Big-Endian streams, so we need to do it manually.
byte[] encodedSize = d.GetFieldable("compressed").GetBinaryValue().Take(4).Reverse().ToArray();
int actualSize = BitConverter.ToInt32(encodedSize, 0);
int compressedSize = int.Parse(d.Get("compressedSize"));
bool binary = int.Parse(d.Get("id")) % 2 > 0;
int shouldSize = shouldStillBeCompressed
? compressedSize
: (binary ? BINARY_PLAIN_LENGTH : TEXT_PLAIN_LENGTH);
Assert.AreEqual(shouldSize, actualSize, "size incorrect");
if (!shouldStillBeCompressed)
{
Assert.IsFalse(compressedSize == actualSize,
"uncompressed field should have another size than recorded in index");
}
}
}
Assert.AreEqual(34 * 2, count, "correct number of tests");
}
finally
{
reader.Dispose();
}
}
internal void WriteField(FieldInfo fi, IFieldable field)
{
fieldsStream.WriteVInt(fi.number);
byte bits = 0;
if (field.IsTokenized)
bits |= FieldsWriter.FIELD_IS_TOKENIZED;
if (field.IsBinary)
bits |= FieldsWriter.FIELD_IS_BINARY;
fieldsStream.WriteByte(bits);
// compression is disabled for the current field
if (field.IsBinary)
{
byte[] data = field.GetBinaryValue();
int len = field.BinaryLength;
int offset = field.BinaryOffset;
fieldsStream.WriteVInt(len);
fieldsStream.WriteBytes(data, offset, len);
}
else
{
fieldsStream.WriteString(field.StringValue);
}
}