public SimpleTextStoredFieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) { _fieldInfos = fn; var success = false; try { _input = directory.OpenInput( IndexFileNames.SegmentFileName(si.Name, "", SimpleTextStoredFieldsWriter.FIELDS_EXTENSION), context); success = true; } finally { if (!success) { try { Dispose(); } catch { // ensure we throw our original exception } } } ReadIndex(si.DocCount); }
public BinaryDocValuesAnonymousInnerClassHelper(SimpleTextDocValuesReader outerInstance, OneField field, IndexInput input, BytesRef scratch) { _outerInstance = outerInstance; _field = field; _input = input; _scratch = scratch; }
public NumericDocValuesAnonymousInnerClassHelper(SimpleTextDocValuesReader outerInstance, OneField field, IndexInput @in, BytesRef scratch) { _outerInstance = outerInstance; _field = field; _input = @in; _scratch = scratch; }
public SortedDocValuesAnonymousInnerClassHelper(SimpleTextDocValuesReader outerInstance, OneField field, IndexInput input, BytesRef scratch) { _outerInstance = outerInstance; _field = field; _input = input; _scratch = scratch; _decoderFormat = field.Pattern; _ordDecoderFormat = field.OrdPattern; }
internal TermsReader(FSTTermsReader outerInstance, FieldInfo fieldInfo, IndexInput @in, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) { this.outerInstance = outerInstance; this.fieldInfo = fieldInfo; this.numTerms = numTerms; this.sumTotalTermFreq = sumTotalTermFreq; this.sumDocFreq = sumDocFreq; this.docCount = docCount; this.longsSize = longsSize; this.dict = new FST <FSTTermOutputs.TermData>(@in, new FSTTermOutputs(fieldInfo, longsSize)); }
public SortedDocValuesAnonymousInnerClassHelper(SimpleTextDocValuesReader outerInstance, Lucene.Net.Codecs.SimpleText.SimpleTextDocValuesReader.OneField field, IndexInput @in, BytesRef scratch, DecimalFormat decoder, DecimalFormat ordDecoder) { this.outerInstance = outerInstance; this.field = field; this.@in = @in; this.scratch = scratch; this.decoder = decoder; this.ordDecoder = ordDecoder; }
public SortedSetDocValuesAnonymousInnerClassHelper(SimpleTextDocValuesReader outerInstance, OneField field, IndexInput input, BytesRef scratch) { _outerInstance = outerInstance; _field = field; _input = input; _scratch = scratch; _decoderFormat = field.Pattern; _currentOrds = new string[0]; _currentIndex = 0; }
private void SeekDir(IndexInput @in) { if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) { @in.Seek(@in.Length() - CodecUtil.FooterLength() - 8); } else { @in.Seek(@in.Length() - 8); } @in.Seek(@in.ReadLong()); }
public SortedSetDocValuesAnonymousInnerClassHelper(SimpleTextDocValuesReader outerInstance, Lucene.Net.Codecs.SimpleText.SimpleTextDocValuesReader.OneField field, IndexInput @in, BytesRef scratch, DecimalFormat decoder) { this.outerInstance = outerInstance; this.field = field; this.@in = @in; this.scratch = scratch; this.decoder = decoder; currentOrds = new string[0]; currentIndex = 0; }
public override SortedSetDocValues GetSortedSet(FieldInfo fieldInfo) { OneField field = FIELDS[fieldInfo.Name]; // SegmentCoreReaders already verifies this field is // valid: Debug.Assert(field != null); IndexInput @in = (IndexInput)DATA.Clone(); BytesRef scratch = new BytesRef(); DecimalFormat decoder = new DecimalFormat(field.Pattern, new DecimalFormatSymbols(Locale.ROOT)); return(new SortedSetDocValuesAnonymousInnerClassHelper(this, field, @in, scratch, decoder)); }
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) { string termsFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, FSTTermsWriter.TERMS_EXTENSION); this.postingsReader = postingsReader; IndexInput @in = state.Directory.OpenInput(termsFileName, state.Context); bool success = false; try { version = ReadHeader(@in); if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) { CodecUtil.ChecksumEntireFile(@in); } this.postingsReader.Init(@in); SeekDir(@in); FieldInfos fieldInfos = state.FieldInfos; int numFields = @in.ReadVInt(); for (int i = 0; i < numFields; i++) { int fieldNumber = @in.ReadVInt(); FieldInfo fieldInfo = fieldInfos.FieldInfo(fieldNumber); long numTerms = @in.ReadVLong(); long sumTotalTermFreq = fieldInfo.FieldIndexOptions == IndexOptions.DOCS_ONLY ? -1 : @in.ReadVLong(); long sumDocFreq = @in.ReadVLong(); int docCount = @in.ReadVInt(); int longsSize = @in.ReadVInt(); TermsReader current = new TermsReader(this, fieldInfo, @in, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize); TermsReader previous = fields[fieldInfo.Name] = current; CheckFieldSummary(state.SegmentInfo, @in, current, previous); } success = true; } finally { if (success) { IOUtils.Close(@in); } else { IOUtils.CloseWhileHandlingException(@in); } } }
protected override void Dispose(bool disposing) { if (disposing) { return; } try { IOUtils.Close(_input); } finally { _input = null; _offsets = null; } }
private void CheckFieldSummary(SegmentInfo info, IndexInput @in, TermsReader field, TermsReader previous) { // #docs with field must be <= #docs if (field.docCount < 0 || field.docCount > info.DocCount) { throw new CorruptIndexException("invalid docCount: " + field.docCount + " maxDoc: " + info.DocCount + " (resource=" + @in + ")"); } // #postings must be >= #docs with field if (field.sumDocFreq < field.docCount) { throw new CorruptIndexException("invalid sumDocFreq: " + field.sumDocFreq + " docCount: " + field.docCount + " (resource=" + @in + ")"); } // #positions must be >= #postings if (field.sumTotalTermFreq != -1 && field.sumTotalTermFreq < field.sumDocFreq) { throw new CorruptIndexException("invalid sumTotalTermFreq: " + field.sumTotalTermFreq + " sumDocFreq: " + field.sumDocFreq + " (resource=" + @in + ")"); } if (previous != null) { throw new CorruptIndexException("duplicate fields: " + field.fieldInfo.Name + " (resource=" + @in + ")"); } }
public SimpleTextFieldsReader(SegmentReadState state) { _maxDoc = state.SegmentInfo.DocCount; _fieldInfos = state.FieldInfos; _input = state.Directory.OpenInput( SimpleTextPostingsFormat.GetPostingsFileName(state.SegmentInfo.Name, state.SegmentSuffix), state.Context); bool success = false; try { _fields = ReadFields((IndexInput)_input.Clone()); success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(); } } }
private SortedDictionary<string, long?> ReadFields(IndexInput @in) { ChecksumIndexInput input = new BufferedChecksumIndexInput(@in); var scratch = new BytesRef(10); var fields = new SortedDictionary<string, long?>(); while (true) { SimpleTextUtil.ReadLine(input, scratch); if (scratch.Equals(SimpleTextFieldsWriter.END)) { SimpleTextUtil.CheckFooter(input); return fields; } if (StringHelper.StartsWith(scratch, SimpleTextFieldsWriter.FIELD)) { var fieldName = scratch.Bytes.SubList(scratch.Offset + SimpleTextFieldsWriter.FIELD.Length, scratch.Length - SimpleTextFieldsWriter.FIELD.Length).ToString(); fields[fieldName] = input.FilePointer; } } }
private SortedDictionary <string, long?> ReadFields(IndexInput @in) { ChecksumIndexInput input = new BufferedChecksumIndexInput(@in); var scratch = new BytesRef(10); var fields = new SortedDictionary <string, long?>(); while (true) { SimpleTextUtil.ReadLine(input, scratch); if (scratch.Equals(SimpleTextFieldsWriter.END)) { SimpleTextUtil.CheckFooter(input); return(fields); } if (StringHelper.StartsWith(scratch, SimpleTextFieldsWriter.FIELD)) { var fieldName = Encoding.UTF8.GetString(scratch.Bytes, scratch.Offset + SimpleTextFieldsWriter.FIELD.Length, scratch.Length - SimpleTextFieldsWriter.FIELD.Length); fields[fieldName] = input.FilePointer; } } }
public SimpleTextTermVectorsReader(Directory directory, SegmentInfo si, IOContext context) { bool success = false; try { _input = directory.OpenInput(IndexFileNames.SegmentFileName(si.Name, "", VECTORS_EXTENSION), context); success = true; } finally { if (!success) { try { Dispose(); } // ensure we throw our original exception catch (Exception) { } } } ReadIndex(si.DocCount); }
public SimpleTextDocsEnum(SimpleTextFieldsReader outerInstance) { _inStart = outerInstance._input; _in = (IndexInput) _inStart.Clone(); }
protected internal VariableIntBlockIndexInput(IndexInput input) { _input = input; MAX_BLOCK_SIZE = input.ReadInt(); }
public FixedIntBlockIndexInput(IndexInput @in) { _input = @in; BLOCK_SIZE = @in.ReadVInt(); }
protected internal abstract IBlockReader GetBlockReader(IndexInput @in, int[] buffer);
public NumericDocValuesAnonymousInnerClassHelper(SimpleTextDocValuesReader outerInstance, OneField field, IndexInput input, BytesRef scratch) { _outerInstance = outerInstance; _field = field; _input = input; _scratch = scratch; }
// used by clone internal SimpleTextTermVectorsReader(long[] offsets, IndexInput @in) { this._offsets = offsets; _input = @in; }
public SimpleTextDocValuesReader(SegmentReadState state, string ext) { DATA = state.Directory.OpenInput( IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, ext), state.Context); MAX_DOC = state.SegmentInfo.DocCount; while (true) { ReadLine(); if (SCRATCH.Equals(SimpleTextDocValuesWriter.END)) { break; } Debug.Assert(StartsWith(SimpleTextDocValuesWriter.FIELD), SCRATCH.Utf8ToString()); var fieldName = StripPrefix(SimpleTextDocValuesWriter.FIELD); var field = new OneField(); FIELDS[fieldName] = field; ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.TYPE), SCRATCH.Utf8ToString()); var dvType = (FieldInfo.DocValuesType_e) Enum.Parse(typeof(FieldInfo.DocValuesType_e), StripPrefix(SimpleTextDocValuesWriter.TYPE)); if (dvType == FieldInfo.DocValuesType_e.NUMERIC) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MINVALUE), "got " + SCRATCH.Utf8ToString() + " field=" + fieldName + " ext=" + ext); field.MinValue = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.MINVALUE)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (1 + field.Pattern.Length + 2) * MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.BINARY) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength + 2) * MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.SORTED || dvType == FieldInfo.DocValuesType_e.SORTED_SET) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.NUMVALUES)); field.NumValues = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.NUMVALUES)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.ORDPATTERN)); field.OrdPattern = StripPrefix(SimpleTextDocValuesWriter.ORDPATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength) * field.NumValues + (1 + field.OrdPattern.Length) * MAX_DOC); } else { throw new InvalidEnumArgumentException(); } } // We should only be called from above if at least one // field has DVs: Debug.Assert(FIELDS.Count > 0); }
public SimpleTextDocValuesReader(SegmentReadState state, string ext) { DATA = state.Directory.OpenInput( IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, ext), state.Context); MAX_DOC = state.SegmentInfo.DocCount; while (true) { ReadLine(); if (SCRATCH.Equals(SimpleTextDocValuesWriter.END)) { break; } Debug.Assert(StartsWith(SimpleTextDocValuesWriter.FIELD), SCRATCH.Utf8ToString()); var fieldName = StripPrefix(SimpleTextDocValuesWriter.FIELD); var field = new OneField(); FIELDS[fieldName] = field; ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.TYPE), SCRATCH.Utf8ToString()); var dvType = (FieldInfo.DocValuesType_e) Enum.Parse(typeof (FieldInfo.DocValuesType_e), StripPrefix(SimpleTextDocValuesWriter.TYPE)); if (dvType == FieldInfo.DocValuesType_e.NUMERIC) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MINVALUE), "got " + SCRATCH.Utf8ToString() + " field=" + fieldName + " ext=" + ext); field.MinValue = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.MINVALUE)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (1 + field.Pattern.Length + 2)*MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.BINARY) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength + 2)*MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.SORTED || dvType == FieldInfo.DocValuesType_e.SORTED_SET) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.NUMVALUES)); field.NumValues = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.NUMVALUES)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.ORDPATTERN)); field.OrdPattern = StripPrefix(SimpleTextDocValuesWriter.ORDPATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength)*field.NumValues + (1 + field.OrdPattern.Length)*MAX_DOC); } else { throw new InvalidEnumArgumentException(); } } // We should only be called from above if at least one // field has DVs: Debug.Assert(FIELDS.Count > 0); }
// used by clone internal SimpleTextTermVectorsReader(long[] offsets, IndexInput input) { _offsets = offsets; _input = input; }
public bool CanReuse(IndexInput @in) { return @in == _inStart; }
public bool CanReuse(IndexInput @in) { return(@in == _inStart); }
private long[] _offsets; // docid -> offset in .vec file #endregion Fields #region Constructors public SimpleTextTermVectorsReader(Directory directory, SegmentInfo si, IOContext context) { bool success = false; try { _input = directory.OpenInput(IndexFileNames.SegmentFileName(si.Name, "", SimpleTextTermVectorsWriter.VECTORS_EXTENSION), context); success = true; } finally { if (!success) { try { Dispose(); } catch (Exception) { // ensure we throw our original exception } } } ReadIndex(si.DocCount); }
/// <remarks>Used by clone</remarks> internal SimpleTextStoredFieldsReader(long[] offsets, IndexInput input, FieldInfos fieldInfos) { _offsets = offsets; _input = input; _fieldInfos = fieldInfos; }
public SimpleTextDocsAndPositionsEnum(SimpleTextFieldsReader outerInstance) { _inStart = outerInstance._input; _in = (IndexInput)_inStart.Clone(); }
protected override void Dispose(bool disposing) { if (disposing) return; try { IOUtils.Close(_input); } finally { _input = null; _offsets = null; } }
private int ReadHeader(IndexInput @in) { return(CodecUtil.CheckHeader(@in, FSTTermsWriter.TERMS_CODEC_NAME, FSTTermsWriter.TERMS_VERSION_START, FSTTermsWriter.TERMS_VERSION_CURRENT)); }