public override void Read(DataInput indexIn, bool absolute)
{
if (absolute)
{
fp = indexIn.ReadVLong();
}
else
{
fp += indexIn.ReadVLong();
}
}
public override bool Load(DataInput input)
{
lock (this)
{
count = input.ReadVLong();
this.higherWeightsCompletion = new FSTCompletion(new FST<object>(input, NoOutputs.Singleton));
this.normalCompletion = new FSTCompletion(higherWeightsCompletion.FST, false, exactMatchFirst);
return true;
}
}
// pre-order traversal
private void ReadRecursively(DataInput @in, TernaryTreeNode node)
{
node.splitchar = @in.ReadString().First();
sbyte mask = (sbyte)@in.ReadByte();
if ((mask & HAS_TOKEN) != 0)
{
node.token = @in.ReadString();
}
if ((mask & HAS_VALUE) != 0)
{
node.val = Convert.ToInt64(@in.ReadLong());
}
if ((mask & LO_KID) != 0)
{
node.loKid = new TernaryTreeNode();
ReadRecursively(@in, node.loKid);
}
if ((mask & EQ_KID) != 0)
{
node.eqKid = new TernaryTreeNode();
ReadRecursively(@in, node.eqKid);
}
if ((mask & HI_KID) != 0)
{
node.hiKid = new TernaryTreeNode();
ReadRecursively(@in, node.hiKid);
}
}
public override bool Load(DataInput input)
{
lock (this)
{
count = input.ReadVLong();
root = new TernaryTreeNode();
ReadRecursively(input, root);
return true;
}
}
public override void Read(DataInput indexIn, bool absolute)
{
if (absolute)
{
upto = indexIn.ReadVInt();
fp = indexIn.ReadVLong();
}
else
{
int uptoDelta = indexIn.ReadVInt();
if ((uptoDelta & 1) == 1)
{
// same block
upto += (int)((uint)uptoDelta >> 1);
}
else
{
// new block
upto = (int)((uint)uptoDelta >> 1);
fp += indexIn.ReadVLong();
}
}
// TODO: we can't do this assert because non-causal
// int encoders can have upto over the buffer size
//assert upto < maxBlockSize: "upto=" + upto + " max=" + maxBlockSize;
}
private void ReadRecursively(DataInput @in, JaspellTernarySearchTrie.TSTNode node)
{
node.splitchar = @in.ReadString()[0];
sbyte mask = (sbyte)@in.ReadByte();
if ((mask & HAS_VALUE) != 0)
{
node.data = Convert.ToInt64(@in.ReadLong());
}
if ((mask & LO_KID) != 0)
{
var kid = new JaspellTernarySearchTrie.TSTNode(trie, '\0', node);
node.relatives[JaspellTernarySearchTrie.TSTNode.LOKID] = kid;
ReadRecursively(@in, kid);
}
if ((mask & EQ_KID) != 0)
{
var kid = new JaspellTernarySearchTrie.TSTNode(trie, '\0', node);
node.relatives[JaspellTernarySearchTrie.TSTNode.EQKID] = kid;
ReadRecursively(@in, kid);
}
if ((mask & HI_KID) != 0)
{
var kid = new JaspellTernarySearchTrie.TSTNode(trie, '\0', node);
node.relatives[JaspellTernarySearchTrie.TSTNode.HIKID] = kid;
ReadRecursively(@in, kid);
}
}
public override bool Load(DataInput input)
{
count = input.ReadVLong();
var root = new JaspellTernarySearchTrie.TSTNode(trie, '\0', null);
ReadRecursively(input, root);
trie.Root = root;
return true;
}
private void _decodeTerm(DataInput @in, FieldInfo fieldInfo, Lucene41PostingsWriter.IntBlockTermState termState)
{
bool fieldHasPositions = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
bool fieldHasOffsets = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
bool fieldHasPayloads = fieldInfo.HasPayloads();
if (termState.DocFreq == 1)
{
termState.SingletonDocID = @in.ReadVInt();
}
else
{
termState.SingletonDocID = -1;
termState.DocStartFP += @in.ReadVLong();
}
if (fieldHasPositions)
{
termState.PosStartFP += @in.ReadVLong();
if (termState.TotalTermFreq > Lucene41PostingsFormat.BLOCK_SIZE)
{
termState.LastPosBlockOffset = @in.ReadVLong();
}
else
{
termState.LastPosBlockOffset = -1;
}
if ((fieldHasPayloads || fieldHasOffsets) && termState.TotalTermFreq >= Lucene41PostingsFormat.BLOCK_SIZE)
{
termState.PayStartFP += @in.ReadVLong();
}
}
if (termState.DocFreq > Lucene41PostingsFormat.BLOCK_SIZE)
{
termState.SkipOffset = @in.ReadVLong();
}
else
{
termState.SkipOffset = -1;
}
}
/// <summary>
/// Reads and validates a header previously written with
/// <seealso cref="#writeHeader(DataOutput, String, int)"/>.
/// <p>
/// When reading a file, supply the expected <code>codec</code> and
/// an expected version range (<code>minVersion to maxVersion</code>).
/// </summary>
/// <param name="in"> Input stream, positioned at the point where the
/// header was previously written. Typically this is located
/// at the beginning of the file. </param>
/// <param name="codec"> The expected codec name. </param>
/// <param name="minVersion"> The minimum supported expected version number. </param>
/// <param name="maxVersion"> The maximum supported expected version number. </param>
/// <returns> The actual version found, when a valid header is found
/// that matches <code>codec</code>, with an actual version
/// where <code>minVersion <= actual <= maxVersion</code>.
/// Otherwise an exception is thrown. </returns>
/// <exception cref="CorruptIndexException"> If the first four bytes are not
/// <seealso cref="#CODEC_MAGIC"/>, or if the actual codec found is
/// not <code>codec</code>. </exception>
/// <exception cref="IndexFormatTooOldException"> If the actual version is less
/// than <code>minVersion</code>. </exception>
/// <exception cref="IndexFormatTooNewException"> If the actual version is greater
/// than <code>maxVersion</code>. </exception>
/// <exception cref="IOException"> If there is an I/O error reading from the underlying medium. </exception>
/// <seealso cref= #writeHeader(DataOutput, String, int) </seealso>
public static int CheckHeader(DataInput @in, string codec, int minVersion, int maxVersion)
{
// Safety to guard against reading a bogus string:
int actualHeader = @in.ReadInt();
if (actualHeader != CODEC_MAGIC)
{
throw new System.IO.IOException("codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CODEC_MAGIC + " (resource: " + @in + ")");
}
return CheckHeaderNoMagic(@in, codec, minVersion, maxVersion);
}
public override void CopyBytes(DataInput input, long numBytes)
{
CheckCrashed();
CheckDiskFull(null, 0, input, numBytes);
@delegate.CopyBytes(input, numBytes);
Dir.MaybeThrowDeterministicException();
}
private void CheckDiskFull(byte[] b, int offset, DataInput @in, long len)
{
long freeSpace = Dir.MaxSize == 0 ? 0 : Dir.MaxSize - Dir.SizeInBytes();
long realUsage = 0;
// Enforce disk full:
if (Dir.MaxSize != 0 && freeSpace <= len)
{
// Compute the real disk free. this will greatly slow
// down our test but makes it more accurate:
realUsage = Dir.RecomputedActualSizeInBytes;
freeSpace = Dir.MaxSize - realUsage;
}
if (Dir.MaxSize != 0 && freeSpace <= len)
{
if (freeSpace > 0)
{
realUsage += freeSpace;
if (b != null)
{
@delegate.WriteBytes(b, offset, (int)freeSpace);
}
else
{
@delegate.CopyBytes(@in, len);
}
}
if (realUsage > Dir.MaxUsedSize)
{
Dir.MaxUsedSize = realUsage;
}
string message = "fake disk full at " + Dir.RecomputedActualSizeInBytes + " bytes when writing " + Name + " (file length=" + @delegate.Length;
if (freeSpace > 0)
{
message += "; wrote " + freeSpace + " of " + len + " bytes";
}
message += ")";
/*if (LuceneTestCase.VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": MDW: now throw fake disk full");
(new Exception()).printStackTrace(System.out);
}*/
throw new System.IO.IOException(message);
}
}
/// <summary>
/// Copy numBytes bytes from input to ourself. </summary>
public virtual void CopyBytes(DataInput input, long numBytes)
{
Debug.Assert(numBytes >= 0, "numBytes=" + numBytes);
long left = numBytes;
if (CopyBuffer == null)
{
CopyBuffer = new byte[COPY_BUFFER_SIZE];
}
while (left > 0)
{
int toCopy;
if (left > COPY_BUFFER_SIZE)
{
toCopy = COPY_BUFFER_SIZE;
}
else
{
toCopy = (int)left;
}
input.ReadBytes(CopyBuffer, 0, toCopy);
WriteBytes(CopyBuffer, 0, toCopy);
left -= toCopy;
}
}
public override void DecodeTerm(long[] empty, DataInput input, FieldInfo fieldInfo, BlockTermState _termState,
bool absolute)
{
PulsingTermState termState = (PulsingTermState) _termState;
Debug.Debug.Assert((empty.Length == 0);
termState.Absolute = termState.Absolute || absolute;
// if we have positions, its total TF, otherwise its computed based on docFreq.
// TODO Double check this is right..
long count = FieldInfo.IndexOptions_e.DOCS_AND_FREQS_AND_POSITIONS.CompareTo(fieldInfo.IndexOptions) <= 0
? termState.TotalTermFreq
: termState.DocFreq;
//System.out.println(" count=" + count + " threshold=" + maxPositions);
if (count <= maxPositions)
{
// Inlined into terms dict -- just read the byte[] blob in,
// but don't decode it now (we only decode when a DocsEnum
// or D&PEnum is pulled):
termState.PostingsSize = input.ReadVInt();
if (termState.Postings == null || termState.Postings.Length < termState.PostingsSize)
{
termState.Postings = new byte[ArrayUtil.Oversize(termState.PostingsSize, 1)];
}
// TODO: sort of silly to copy from one big byte[]
// (the blob holding all inlined terms' blobs for
// current term block) into another byte[] (just the
// blob for this term)...
input.ReadBytes(termState.Postings, 0, termState.PostingsSize);
//System.out.println(" inlined bytes=" + termState.postingsSize);
termState.Absolute = termState.Absolute || absolute;
}
else
{
int longsSize = fields == null ? 0 : fields[fieldInfo.Number];
if (termState.Longs == null)
{
termState.Longs = new long[longsSize];
}
for (int i = 0; i < longsSize; i++)
{
termState.Longs[i] = input.ReadVLong();
}
termState.PostingsSize = -1;
termState.WrappedTermState.DocFreq = termState.DocFreq;
termState.WrappedTermState.TotalTermFreq = termState.TotalTermFreq;
_wrappedPostingsReader.DecodeTerm(termState.Longs, input, fieldInfo,
termState.WrappedTermState,
termState.Absolute);
termState.Absolute = false;
}
}
public override bool Load(DataInput @out)
{
return false;
}
public override void Read(DataInput indexIn, bool absolute)
{
if (absolute)
{
upto = indexIn.ReadVInt();
fp = indexIn.ReadVLong();
}
else
{
int uptoDelta = indexIn.ReadVInt();
if ((uptoDelta & 1) == 1)
{
// same block
upto += (int)((uint)uptoDelta >> 1);
}
else
{
// new block
upto = (int)((uint)uptoDelta >> 1);
fp += indexIn.ReadVLong();
}
}
Debug.Assert(upto < outerInstance.blockSize);
}
/// <summary>
/// Like {@link
/// #checkHeader(DataInput,String,int,int)} except this
/// version assumes the first int has already been read
/// and validated from the input.
/// </summary>
public static int CheckHeaderNoMagic(DataInput @in, string codec, int minVersion, int maxVersion)
{
string actualCodec = @in.ReadString();
if (!actualCodec.Equals(codec))
{
throw new System.IO.IOException("codec mismatch: actual codec=" + actualCodec + " vs expected codec=" + codec + " (resource: " + @in + ")");
}
int actualVersion = @in.ReadInt();
if (actualVersion < minVersion)
{
throw new System.IO.IOException("Version: " + actualVersion + " is not supported. Minimum Version number is " + minVersion + ".");
}
if (actualVersion > maxVersion)
{
throw new System.IO.IOException("Version: " + actualVersion + " is not supported. Maximum Version number is " + maxVersion + ".");
}
return actualVersion;
}
/// <summary>
/// Restore a <seealso cref="ForUtil"/> from a <seealso cref="DataInput"/>.
/// </summary>
public ForUtil(DataInput @in)
{
int packedIntsVersion = @in.ReadVInt();
PackedInts.CheckVersion(packedIntsVersion);
EncodedSizes = new int[33];
Encoders = new PackedInts.Encoder[33];
Decoders = new PackedInts.Decoder[33];
Iterations = new int[33];
for (int bpv = 1; bpv <= 32; ++bpv)
{
int code = @in.ReadVInt();
int formatId = (int)((uint)code >> 5);
int bitsPerValue = (code & 31) + 1;
PackedInts.Format format = PackedInts.Format.ById(formatId);
Debug.Assert(format.IsSupported(bitsPerValue));
EncodedSizes[bpv] = EncodedSize(format, packedIntsVersion, bitsPerValue);
Encoders[bpv] = PackedInts.GetEncoder(format, packedIntsVersion, bitsPerValue);
Decoders[bpv] = PackedInts.GetDecoder(format, packedIntsVersion, bitsPerValue);
Iterations[bpv] = ComputeIterations(Decoders[bpv]);
}
}
public override void DecodeTerm(long[] longs, DataInput @in, FieldInfo fieldInfo, BlockTermState _termState, bool absolute)
{
Lucene41PostingsWriter.IntBlockTermState termState = (Lucene41PostingsWriter.IntBlockTermState)_termState;
bool fieldHasPositions = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
bool fieldHasOffsets = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
bool fieldHasPayloads = fieldInfo.HasPayloads();
if (absolute)
{
termState.DocStartFP = 0;
termState.PosStartFP = 0;
termState.PayStartFP = 0;
}
if (Version < Lucene41PostingsWriter.VERSION_META_ARRAY) // backward compatibility
{
_decodeTerm(@in, fieldInfo, termState);
return;
}
termState.DocStartFP += longs[0];
if (fieldHasPositions)
{
termState.PosStartFP += longs[1];
if (fieldHasOffsets || fieldHasPayloads)
{
termState.PayStartFP += longs[2];
}
}
if (termState.DocFreq == 1)
{
termState.SingletonDocID = @in.ReadVInt();
}
else
{
termState.SingletonDocID = -1;
}
if (fieldHasPositions)
{
if (termState.TotalTermFreq > Lucene41PostingsFormat.BLOCK_SIZE)
{
termState.LastPosBlockOffset = @in.ReadVLong();
}
else
{
termState.LastPosBlockOffset = -1;
}
}
if (termState.DocFreq > Lucene41PostingsFormat.BLOCK_SIZE)
{
termState.SkipOffset = @in.ReadVLong();
}
else
{
termState.SkipOffset = -1;
}
}
