private void AddField(Document doc, FieldInfo fi, bool binary, bool compressed, bool tokenize, IState state)
{
//we have a binary stored field, and it may be compressed
if (binary)
{
int toRead = fieldsStream.ReadVInt(state);
var b = new byte[toRead];
fieldsStream.ReadBytes(b, 0, b.Length, state);
doc.Add(compressed ? new Field(fi.name, Uncompress(b), Field.Store.YES) : new Field(fi.name, b, Field.Store.YES));
}
else
{
const Field.Store store = Field.Store.YES;
Field.Index index = FieldExtensions.ToIndex(fi.isIndexed, tokenize);
Field.TermVector termVector = FieldExtensions.ToTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector);
AbstractField f;
if (compressed)
{
int toRead = fieldsStream.ReadVInt(state);
var b = new byte[toRead];
fieldsStream.ReadBytes(b, 0, b.Length, state);
f = new Field(fi.name, false, System.Text.Encoding.GetEncoding("UTF-8").GetString(Uncompress(b)), store, index,
termVector)
{
OmitTermFreqAndPositions = fi.omitTermFreqAndPositions, OmitNorms = fi.omitNorms
};
}
else
{
f = new Field(fi.name, false, fieldsStream.ReadString(state), store, index, termVector)
{
OmitTermFreqAndPositions = fi.omitTermFreqAndPositions, OmitNorms = fi.omitNorms
};
}
doc.Add(f);
}
}
private static IDictionary <string, FileEntry> ReadLegacyEntries(IndexInput stream, int firstInt)
{
IDictionary <string, FileEntry> entries = new Dictionary <string, FileEntry>();
int count;
bool stripSegmentName;
if (firstInt < CompoundFileWriter.FORMAT_PRE_VERSION)
{
if (firstInt < CompoundFileWriter.FORMAT_NO_SEGMENT_PREFIX)
{
throw new CorruptIndexException("Incompatible format version: "
+ firstInt + " expected >= " + CompoundFileWriter.FORMAT_NO_SEGMENT_PREFIX + " (resource: " + stream + ")");
}
// It's a post-3.1 index, read the count.
count = stream.ReadVInt32();
stripSegmentName = false;
}
else
{
count = firstInt;
stripSegmentName = true;
}
// read the directory and init files
long streamLength = stream.Length;
FileEntry entry = null;
for (int i = 0; i < count; i++)
{
long offset = stream.ReadInt64();
if (offset < 0 || offset > streamLength)
{
throw new CorruptIndexException("Invalid CFS entry offset: " + offset + " (resource: " + stream + ")");
}
string id = stream.ReadString();
if (stripSegmentName)
{
// Fix the id to not include the segment names. this is relevant for
// pre-3.1 indexes.
id = IndexFileNames.StripSegmentName(id);
}
if (entry != null)
{
// set length of the previous entry
entry.Length = offset - entry.Offset;
}
entry = new FileEntry();
entry.Offset = offset;
FileEntry previous = entries.Put(id, entry);
if (previous != null)
{
throw new CorruptIndexException("Duplicate cfs entry id=" + id + " in CFS: " + stream);
}
}
// set the length of the final entry
if (entry != null)
{
entry.Length = streamLength - entry.Offset;
}
return(entries);
}
public override string ReadString()
{
EnsureOpen();
return(@delegate.ReadString());
}
private static IDictionary<string, FileEntry> ReadLegacyEntries(IndexInput stream, int firstInt)
{
IDictionary<string, FileEntry> entries = new Dictionary<string, FileEntry>();
int count;
bool stripSegmentName;
if (firstInt < CompoundFileWriter.FORMAT_PRE_VERSION)
{
if (firstInt < CompoundFileWriter.FORMAT_NO_SEGMENT_PREFIX)
{
throw new CorruptIndexException("Incompatible format version: " + firstInt + " expected >= " + CompoundFileWriter.FORMAT_NO_SEGMENT_PREFIX + " (resource: " + stream + ")");
}
// It's a post-3.1 index, read the count.
count = stream.ReadVInt();
stripSegmentName = false;
}
else
{
count = firstInt;
stripSegmentName = true;
}
// read the directory and init files
long streamLength = stream.Length();
FileEntry entry = null;
for (int i = 0; i < count; i++)
{
long offset = stream.ReadLong();
if (offset < 0 || offset > streamLength)
{
throw new CorruptIndexException("Invalid CFS entry offset: " + offset + " (resource: " + stream + ")");
}
string id = stream.ReadString();
if (stripSegmentName)
{
// Fix the id to not include the segment names. this is relevant for
// pre-3.1 indexes.
id = IndexFileNames.StripSegmentName(id);
}
if (entry != null)
{
// set length of the previous entry
entry.Length = offset - entry.Offset;
}
entry = new FileEntry();
entry.Offset = offset;
FileEntry previous = entries[id] = entry;
if (previous != null)
{
throw new CorruptIndexException("Duplicate cfs entry id=" + id + " in CFS: " + stream);
}
}
// set the length of the final entry
if (entry != null)
{
entry.Length = streamLength - entry.Offset;
}
return entries;
}
/// <summary> Construct a new SegmentInfo instance by reading a
/// previously saved SegmentInfo from input.
///
/// </summary>
/// <param name="dir">directory to load from
/// </param>
/// <param name="format">format of the segments info file
/// </param>
/// <param name="input">input handle to read segment info from
/// </param>
internal SegmentInfo(Directory dir, int format, IndexInput input, IState state)
{
this.dir = dir;
name = input.ReadString(state);
docCount = input.ReadInt(state);
if (format <= SegmentInfos.FORMAT_LOCKLESS)
{
delGen = input.ReadLong(state);
if (format <= SegmentInfos.FORMAT_SHARED_DOC_STORE)
{
docStoreOffset = input.ReadInt(state);
if (docStoreOffset != -1)
{
docStoreSegment = input.ReadString(state);
docStoreIsCompoundFile = (1 == input.ReadByte(state));
}
else
{
docStoreSegment = name;
docStoreIsCompoundFile = false;
}
}
else
{
docStoreOffset = -1;
docStoreSegment = name;
docStoreIsCompoundFile = false;
}
if (format <= SegmentInfos.FORMAT_SINGLE_NORM_FILE)
{
hasSingleNormFile = (1 == input.ReadByte(state));
}
else
{
hasSingleNormFile = false;
}
int numNormGen = input.ReadInt(state);
if (numNormGen == NO)
{
normGen = null;
}
else
{
normGen = new long[numNormGen];
for (int j = 0; j < numNormGen; j++)
{
normGen[j] = input.ReadLong(state);
}
}
isCompoundFile = (sbyte)input.ReadByte(state);
preLockless = (isCompoundFile == CHECK_DIR);
if (format <= SegmentInfos.FORMAT_DEL_COUNT)
{
delCount = input.ReadInt(state);
System.Diagnostics.Debug.Assert(delCount <= docCount);
}
else
{
delCount = -1;
}
if (format <= SegmentInfos.FORMAT_HAS_PROX)
{
hasProx = input.ReadByte(state) == 1;
}
else
{
hasProx = true;
}
if (format <= SegmentInfos.FORMAT_DIAGNOSTICS)
{
diagnostics = input.ReadStringStringMap(state);
}
else
{
diagnostics = new Dictionary <string, string>();
}
}
else
{
delGen = CHECK_DIR;
normGen = null;
isCompoundFile = (sbyte)(CHECK_DIR);
preLockless = true;
hasSingleNormFile = false;
docStoreOffset = -1;
docStoreIsCompoundFile = false;
docStoreSegment = null;
delCount = -1;
hasProx = true;
diagnostics = new Dictionary <string, string>();
}
}