public override void Read(DataInput indexIn, bool absolute) { if (absolute) { fp = indexIn.ReadVLong(); } else { fp += indexIn.ReadVLong(); } }
public override bool Load(DataInput input) { lock (this) { count = input.ReadVLong(); this.higherWeightsCompletion = new FSTCompletion(new FST<object>(input, NoOutputs.Singleton)); this.normalCompletion = new FSTCompletion(higherWeightsCompletion.FST, false, exactMatchFirst); return true; } }
// pre-order traversal private void ReadRecursively(DataInput @in, TernaryTreeNode node) { node.splitchar = @in.ReadString().First(); sbyte mask = (sbyte)@in.ReadByte(); if ((mask & HAS_TOKEN) != 0) { node.token = @in.ReadString(); } if ((mask & HAS_VALUE) != 0) { node.val = Convert.ToInt64(@in.ReadLong()); } if ((mask & LO_KID) != 0) { node.loKid = new TernaryTreeNode(); ReadRecursively(@in, node.loKid); } if ((mask & EQ_KID) != 0) { node.eqKid = new TernaryTreeNode(); ReadRecursively(@in, node.eqKid); } if ((mask & HI_KID) != 0) { node.hiKid = new TernaryTreeNode(); ReadRecursively(@in, node.hiKid); } }
public override bool Load(DataInput input) { lock (this) { count = input.ReadVLong(); root = new TernaryTreeNode(); ReadRecursively(input, root); return true; } }
public override void Read(DataInput indexIn, bool absolute) { if (absolute) { upto = indexIn.ReadVInt(); fp = indexIn.ReadVLong(); } else { int uptoDelta = indexIn.ReadVInt(); if ((uptoDelta & 1) == 1) { // same block upto += (int)((uint)uptoDelta >> 1); } else { // new block upto = (int)((uint)uptoDelta >> 1); fp += indexIn.ReadVLong(); } } // TODO: we can't do this assert because non-causal // int encoders can have upto over the buffer size //assert upto < maxBlockSize: "upto=" + upto + " max=" + maxBlockSize; }
private void ReadRecursively(DataInput @in, JaspellTernarySearchTrie.TSTNode node) { node.splitchar = @in.ReadString()[0]; sbyte mask = (sbyte)@in.ReadByte(); if ((mask & HAS_VALUE) != 0) { node.data = Convert.ToInt64(@in.ReadLong()); } if ((mask & LO_KID) != 0) { var kid = new JaspellTernarySearchTrie.TSTNode(trie, '\0', node); node.relatives[JaspellTernarySearchTrie.TSTNode.LOKID] = kid; ReadRecursively(@in, kid); } if ((mask & EQ_KID) != 0) { var kid = new JaspellTernarySearchTrie.TSTNode(trie, '\0', node); node.relatives[JaspellTernarySearchTrie.TSTNode.EQKID] = kid; ReadRecursively(@in, kid); } if ((mask & HI_KID) != 0) { var kid = new JaspellTernarySearchTrie.TSTNode(trie, '\0', node); node.relatives[JaspellTernarySearchTrie.TSTNode.HIKID] = kid; ReadRecursively(@in, kid); } }
public override bool Load(DataInput input) { count = input.ReadVLong(); var root = new JaspellTernarySearchTrie.TSTNode(trie, '\0', null); ReadRecursively(input, root); trie.Root = root; return true; }
private void _decodeTerm(DataInput @in, FieldInfo fieldInfo, Lucene41PostingsWriter.IntBlockTermState termState) { bool fieldHasPositions = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; bool fieldHasOffsets = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; bool fieldHasPayloads = fieldInfo.HasPayloads(); if (termState.DocFreq == 1) { termState.SingletonDocID = @in.ReadVInt(); } else { termState.SingletonDocID = -1; termState.DocStartFP += @in.ReadVLong(); } if (fieldHasPositions) { termState.PosStartFP += @in.ReadVLong(); if (termState.TotalTermFreq > Lucene41PostingsFormat.BLOCK_SIZE) { termState.LastPosBlockOffset = @in.ReadVLong(); } else { termState.LastPosBlockOffset = -1; } if ((fieldHasPayloads || fieldHasOffsets) && termState.TotalTermFreq >= Lucene41PostingsFormat.BLOCK_SIZE) { termState.PayStartFP += @in.ReadVLong(); } } if (termState.DocFreq > Lucene41PostingsFormat.BLOCK_SIZE) { termState.SkipOffset = @in.ReadVLong(); } else { termState.SkipOffset = -1; } }
/// <summary> /// Reads and validates a header previously written with /// <seealso cref="#writeHeader(DataOutput, String, int)"/>. /// <p> /// When reading a file, supply the expected <code>codec</code> and /// an expected version range (<code>minVersion to maxVersion</code>). /// </summary> /// <param name="in"> Input stream, positioned at the point where the /// header was previously written. Typically this is located /// at the beginning of the file. </param> /// <param name="codec"> The expected codec name. </param> /// <param name="minVersion"> The minimum supported expected version number. </param> /// <param name="maxVersion"> The maximum supported expected version number. </param> /// <returns> The actual version found, when a valid header is found /// that matches <code>codec</code>, with an actual version /// where <code>minVersion <= actual <= maxVersion</code>. /// Otherwise an exception is thrown. </returns> /// <exception cref="CorruptIndexException"> If the first four bytes are not /// <seealso cref="#CODEC_MAGIC"/>, or if the actual codec found is /// not <code>codec</code>. </exception> /// <exception cref="IndexFormatTooOldException"> If the actual version is less /// than <code>minVersion</code>. </exception> /// <exception cref="IndexFormatTooNewException"> If the actual version is greater /// than <code>maxVersion</code>. </exception> /// <exception cref="IOException"> If there is an I/O error reading from the underlying medium. </exception> /// <seealso cref= #writeHeader(DataOutput, String, int) </seealso> public static int CheckHeader(DataInput @in, string codec, int minVersion, int maxVersion) { // Safety to guard against reading a bogus string: int actualHeader = @in.ReadInt(); if (actualHeader != CODEC_MAGIC) { throw new System.IO.IOException("codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CODEC_MAGIC + " (resource: " + @in + ")"); } return CheckHeaderNoMagic(@in, codec, minVersion, maxVersion); }
public override void CopyBytes(DataInput input, long numBytes) { CheckCrashed(); CheckDiskFull(null, 0, input, numBytes); @delegate.CopyBytes(input, numBytes); Dir.MaybeThrowDeterministicException(); }
private void CheckDiskFull(byte[] b, int offset, DataInput @in, long len) { long freeSpace = Dir.MaxSize == 0 ? 0 : Dir.MaxSize - Dir.SizeInBytes(); long realUsage = 0; // Enforce disk full: if (Dir.MaxSize != 0 && freeSpace <= len) { // Compute the real disk free. this will greatly slow // down our test but makes it more accurate: realUsage = Dir.RecomputedActualSizeInBytes; freeSpace = Dir.MaxSize - realUsage; } if (Dir.MaxSize != 0 && freeSpace <= len) { if (freeSpace > 0) { realUsage += freeSpace; if (b != null) { @delegate.WriteBytes(b, offset, (int)freeSpace); } else { @delegate.CopyBytes(@in, len); } } if (realUsage > Dir.MaxUsedSize) { Dir.MaxUsedSize = realUsage; } string message = "fake disk full at " + Dir.RecomputedActualSizeInBytes + " bytes when writing " + Name + " (file length=" + @delegate.Length; if (freeSpace > 0) { message += "; wrote " + freeSpace + " of " + len + " bytes"; } message += ")"; /*if (LuceneTestCase.VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": MDW: now throw fake disk full"); (new Exception()).printStackTrace(System.out); }*/ throw new System.IO.IOException(message); } }
/// <summary> /// Copy numBytes bytes from input to ourself. </summary> public virtual void CopyBytes(DataInput input, long numBytes) { Debug.Assert(numBytes >= 0, "numBytes=" + numBytes); long left = numBytes; if (CopyBuffer == null) { CopyBuffer = new byte[COPY_BUFFER_SIZE]; } while (left > 0) { int toCopy; if (left > COPY_BUFFER_SIZE) { toCopy = COPY_BUFFER_SIZE; } else { toCopy = (int)left; } input.ReadBytes(CopyBuffer, 0, toCopy); WriteBytes(CopyBuffer, 0, toCopy); left -= toCopy; } }
public override void DecodeTerm(long[] empty, DataInput input, FieldInfo fieldInfo, BlockTermState _termState, bool absolute) { PulsingTermState termState = (PulsingTermState) _termState; Debug.Debug.Assert((empty.Length == 0); termState.Absolute = termState.Absolute || absolute; // if we have positions, its total TF, otherwise its computed based on docFreq. // TODO Double check this is right.. long count = FieldInfo.IndexOptions_e.DOCS_AND_FREQS_AND_POSITIONS.CompareTo(fieldInfo.IndexOptions) <= 0 ? termState.TotalTermFreq : termState.DocFreq; //System.out.println(" count=" + count + " threshold=" + maxPositions); if (count <= maxPositions) { // Inlined into terms dict -- just read the byte[] blob in, // but don't decode it now (we only decode when a DocsEnum // or D&PEnum is pulled): termState.PostingsSize = input.ReadVInt(); if (termState.Postings == null || termState.Postings.Length < termState.PostingsSize) { termState.Postings = new byte[ArrayUtil.Oversize(termState.PostingsSize, 1)]; } // TODO: sort of silly to copy from one big byte[] // (the blob holding all inlined terms' blobs for // current term block) into another byte[] (just the // blob for this term)... input.ReadBytes(termState.Postings, 0, termState.PostingsSize); //System.out.println(" inlined bytes=" + termState.postingsSize); termState.Absolute = termState.Absolute || absolute; } else { int longsSize = fields == null ? 0 : fields[fieldInfo.Number]; if (termState.Longs == null) { termState.Longs = new long[longsSize]; } for (int i = 0; i < longsSize; i++) { termState.Longs[i] = input.ReadVLong(); } termState.PostingsSize = -1; termState.WrappedTermState.DocFreq = termState.DocFreq; termState.WrappedTermState.TotalTermFreq = termState.TotalTermFreq; _wrappedPostingsReader.DecodeTerm(termState.Longs, input, fieldInfo, termState.WrappedTermState, termState.Absolute); termState.Absolute = false; } }
public override bool Load(DataInput @out) { return false; }
public override void Read(DataInput indexIn, bool absolute) { if (absolute) { upto = indexIn.ReadVInt(); fp = indexIn.ReadVLong(); } else { int uptoDelta = indexIn.ReadVInt(); if ((uptoDelta & 1) == 1) { // same block upto += (int)((uint)uptoDelta >> 1); } else { // new block upto = (int)((uint)uptoDelta >> 1); fp += indexIn.ReadVLong(); } } Debug.Assert(upto < outerInstance.blockSize); }
/// <summary> /// Like {@link /// #checkHeader(DataInput,String,int,int)} except this /// version assumes the first int has already been read /// and validated from the input. /// </summary> public static int CheckHeaderNoMagic(DataInput @in, string codec, int minVersion, int maxVersion) { string actualCodec = @in.ReadString(); if (!actualCodec.Equals(codec)) { throw new System.IO.IOException("codec mismatch: actual codec=" + actualCodec + " vs expected codec=" + codec + " (resource: " + @in + ")"); } int actualVersion = @in.ReadInt(); if (actualVersion < minVersion) { throw new System.IO.IOException("Version: " + actualVersion + " is not supported. Minimum Version number is " + minVersion + "."); } if (actualVersion > maxVersion) { throw new System.IO.IOException("Version: " + actualVersion + " is not supported. Maximum Version number is " + maxVersion + "."); } return actualVersion; }
/// <summary> /// Restore a <seealso cref="ForUtil"/> from a <seealso cref="DataInput"/>. /// </summary> public ForUtil(DataInput @in) { int packedIntsVersion = @in.ReadVInt(); PackedInts.CheckVersion(packedIntsVersion); EncodedSizes = new int[33]; Encoders = new PackedInts.Encoder[33]; Decoders = new PackedInts.Decoder[33]; Iterations = new int[33]; for (int bpv = 1; bpv <= 32; ++bpv) { int code = @in.ReadVInt(); int formatId = (int)((uint)code >> 5); int bitsPerValue = (code & 31) + 1; PackedInts.Format format = PackedInts.Format.ById(formatId); Debug.Assert(format.IsSupported(bitsPerValue)); EncodedSizes[bpv] = EncodedSize(format, packedIntsVersion, bitsPerValue); Encoders[bpv] = PackedInts.GetEncoder(format, packedIntsVersion, bitsPerValue); Decoders[bpv] = PackedInts.GetDecoder(format, packedIntsVersion, bitsPerValue); Iterations[bpv] = ComputeIterations(Decoders[bpv]); } }
public override void DecodeTerm(long[] longs, DataInput @in, FieldInfo fieldInfo, BlockTermState _termState, bool absolute) { Lucene41PostingsWriter.IntBlockTermState termState = (Lucene41PostingsWriter.IntBlockTermState)_termState; bool fieldHasPositions = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; bool fieldHasOffsets = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; bool fieldHasPayloads = fieldInfo.HasPayloads(); if (absolute) { termState.DocStartFP = 0; termState.PosStartFP = 0; termState.PayStartFP = 0; } if (Version < Lucene41PostingsWriter.VERSION_META_ARRAY) // backward compatibility { _decodeTerm(@in, fieldInfo, termState); return; } termState.DocStartFP += longs[0]; if (fieldHasPositions) { termState.PosStartFP += longs[1]; if (fieldHasOffsets || fieldHasPayloads) { termState.PayStartFP += longs[2]; } } if (termState.DocFreq == 1) { termState.SingletonDocID = @in.ReadVInt(); } else { termState.SingletonDocID = -1; } if (fieldHasPositions) { if (termState.TotalTermFreq > Lucene41PostingsFormat.BLOCK_SIZE) { termState.LastPosBlockOffset = @in.ReadVLong(); } else { termState.LastPosBlockOffset = -1; } } if (termState.DocFreq > Lucene41PostingsFormat.BLOCK_SIZE) { termState.SkipOffset = @in.ReadVLong(); } else { termState.SkipOffset = -1; } }