public SimpleTextDocsEnum Reset(long fp, Bits liveDocs, bool omitTf, int docFreq) { _liveDocs = liveDocs; _in.Seek(fp); _omitTf = omitTf; _docId = -1; _tf = 1; _cost = docFreq; return(this); }
private void SeekDir(IndexInput @in) { if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) { @in.Seek(@in.Length() - CodecUtil.FooterLength() - 8); } else { @in.Seek(@in.Length() - 8); } @in.Seek(@in.ReadLong()); }
public override long Get(int docId) { if (docId < 0 || docId >= _outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (_outerInstance.MAX_DOC - 1) + "; got " + docId); } _input.Seek(_field.DataStartFilePointer + (1 + _field.Pattern.Length + 2) * docId); SimpleTextUtil.ReadLine(_input, _scratch); decimal bd; try { // LUCNENENET: .NET doesn't have a way to specify a pattern with decimal, but all of the standard ones are built in. bd = decimal.Parse(_scratch.Utf8ToString(), NumberStyles.Float, CultureInfo.InvariantCulture); } catch (FormatException ex) { throw new CorruptIndexException("failed to parse long value (resource=" + _input + ")", ex); } SimpleTextUtil.ReadLine(_input, _scratch); // read the line telling us if its real or not return((long)BigInteger.Add(new BigInteger(_field.MinValue), new BigInteger(bd))); }
public override void Get(int docId, BytesRef result) { if (docId < 0 || docId >= _outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (_outerInstance.MAX_DOC - 1) + "; got " + docId); } _input.Seek(_field.DataStartFilePointer + (9 + _field.Pattern.Length + _field.MaxLength + 2) * docId); SimpleTextUtil.ReadLine(_input, _scratch); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextDocValuesWriter.LENGTH)); int len; try { len = int.Parse(_scratch.Bytes.SubList( _scratch.Offset + SimpleTextDocValuesWriter.LENGTH.Length, _scratch.Length - SimpleTextDocValuesWriter.LENGTH.Length).ToString()); } catch (FormatException ex) { throw new CorruptIndexException("failed to parse int value (resource=" + _input + ")", ex); } result.Bytes = new byte[len]; result.Offset = 0; result.Length = len; _input.ReadBytes(result.Bytes, 0, len); }
public override void LookupOrd(long ord, BytesRef result) { if (ord < 0 || ord >= _field.NumValues) { throw new IndexOutOfRangeException("ord must be 0 .. " + (_field.NumValues - 1) + "; got " + ord); } _input.Seek(_field.DataStartFilePointer + ord * (9 + _field.Pattern.Length + _field.MaxLength)); SimpleTextUtil.ReadLine(_input, _scratch); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextDocValuesWriter.LENGTH), "got " + _scratch.Utf8ToString() + " in=" + _input); int len; try { len = (int) _decoderFormat.parse(_scratch.Bytes.SubList( _scratch.Offset + SimpleTextDocValuesWriter.LENGTH.Length, _scratch.Length - SimpleTextDocValuesWriter.LENGTH.Length).ToString()); } catch (Exception pe) { var e = new CorruptIndexException("failed to parse int length (resource=" + _input + ")", pe); throw e; } result.Bytes = new byte[len]; result.Offset = 0; result.Length = len; _input.ReadBytes(result.Bytes, 0, len); }
public override long Get(int docId) { if (docId < 0 || docId >= _outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (_outerInstance.MAX_DOC - 1) + "; got " + docId); } _input.Seek(_field.DataStartFilePointer + (1 + _field.Pattern.Length + 2) * docId); SimpleTextUtil.ReadLine(_input, _scratch); long bd; try { bd = long.Parse(_scratch.Utf8ToString()); } catch (FormatException ex) { throw new CorruptIndexException("failed to parse long value (resource=" + _input + ")", ex); } SimpleTextUtil.ReadLine(_input, _scratch); // read the line telling us if its real or not return(_field.MinValue + bd); }
public bool Get(int index) { _input.Seek(_field.DataStartFilePointer + (9 + _field.Pattern.Length + _field.MaxLength + 2) * index); SimpleTextUtil.ReadLine(_input, _scratch); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextDocValuesWriter.LENGTH)); int len; try { len = int.Parse(_scratch.Bytes.SubList(_scratch.Offset + SimpleTextDocValuesWriter.LENGTH.Length, _scratch.Length - SimpleTextDocValuesWriter.LENGTH.Length).ToString()); } catch (FormatException ex) { throw new CorruptIndexException("failed to parse int value (resource=" + _input + ")", ex); } // skip past bytes var bytes = new byte[len]; _input.ReadBytes(bytes, 0, len); SimpleTextUtil.ReadLine(_input, _scratch); // newline SimpleTextUtil.ReadLine(_input, _scratch); // 'T' or 'F' return(_scratch.Bytes[_scratch.Offset] == (sbyte)'T'); }
public override void Get(int docId, BytesRef result) { if (docId < 0 || docId >= _outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (_outerInstance.MAX_DOC - 1) + "; got " + docId); } _input.Seek(_field.DataStartFilePointer + (9 + _field.Pattern.Length + _field.MaxLength + 2) * docId); SimpleTextUtil.ReadLine(_input, _scratch); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextDocValuesWriter.LENGTH)); int len; try { // LUCNENENET: .NET doesn't have a way to specify a pattern with integer, but all of the standard ones are built in. len = int.Parse(Encoding.UTF8.GetString(_scratch.Bytes, _scratch.Offset + SimpleTextDocValuesWriter.LENGTH.Length, _scratch.Length - SimpleTextDocValuesWriter.LENGTH.Length), NumberStyles.Integer, CultureInfo.InvariantCulture); } catch (FormatException ex) { throw new CorruptIndexException("failed to parse int value (resource=" + _input + ")", ex); } result.Bytes = new byte[len]; result.Offset = 0; result.Length = len; _input.ReadBytes(result.Bytes, 0, len); }
public bool Get(int index) { _input.Seek(_field.DataStartFilePointer + (1 + _field.Pattern.Length + 2) * index); SimpleTextUtil.ReadLine(_input, _scratch); // data SimpleTextUtil.ReadLine(_input, _scratch); // 'T' or 'F' return(_scratch.Bytes[_scratch.Offset] == (sbyte)'T'); }
public override void LookupOrd(long ord, BytesRef result) { if (ord < 0 || ord >= _field.NumValues) { throw new IndexOutOfRangeException("ord must be 0 .. " + (_field.NumValues - 1) + "; got " + ord); } _input.Seek(_field.DataStartFilePointer + ord * (9 + _field.Pattern.Length + _field.MaxLength)); SimpleTextUtil.ReadLine(_input, _scratch); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextDocValuesWriter.LENGTH), "got " + _scratch.Utf8ToString() + " in=" + _input); int len; try { // LUCNENENET: .NET doesn't have a way to specify a pattern with integer, but all of the standard ones are built in. len = int.Parse(Encoding.UTF8.GetString(_scratch.Bytes, _scratch.Offset + SimpleTextDocValuesWriter.LENGTH.Length, _scratch.Length - SimpleTextDocValuesWriter.LENGTH.Length), NumberStyles.Integer, CultureInfo.InvariantCulture); } catch (Exception pe) { var e = new CorruptIndexException("failed to parse int length (resource=" + _input + ")", pe); throw e; } result.Bytes = new byte[len]; result.Offset = 0; result.Length = len; _input.ReadBytes(result.Bytes, 0, len); }
public override void LookupOrd(long ord, BytesRef result) { if (ord < 0 || ord >= field.NumValues) { throw new IndexOutOfRangeException("ord must be 0 .. " + (field.NumValues - 1) + "; got " + ord); } @in.Seek(field.DataStartFilePointer + ord * (9 + field.Pattern.Length + field.MaxLength)); SimpleTextUtil.ReadLine(@in, scratch); Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextDocValuesWriter.LENGTH), "got " + scratch.Utf8ToString() + " in=" + @in); int len; try { len = (int) decoder.parse(scratch.Bytes.SubList( scratch.Offset + SimpleTextDocValuesWriter.LENGTH.Length, scratch.Length - SimpleTextDocValuesWriter.LENGTH.Length).ToString()); } catch (ParseException pe) { CorruptIndexException e = new CorruptIndexException("failed to parse int length (resource=" + @in + ")"); e.initCause(pe); throw e; } result.Bytes = new sbyte[len]; result.Offset = 0; result.Length = len; @in.ReadBytes(result.Bytes, 0, len); }
public override int GetOrd(int docId) { if (docId < 0 || docId >= _outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (_outerInstance.MAX_DOC - 1) + "; got " + docId); } _input.Seek(_field.DataStartFilePointer + _field.NumValues * (9 + _field.Pattern.Length + _field.MaxLength) + docId * (1 + _field.OrdPattern.Length)); SimpleTextUtil.ReadLine(_input, _scratch); try { return(_scratch.Utf8ToString().ToString(_ordDecoderFormat) - 1); } catch (Exception pe) { var e = new CorruptIndexException("failed to parse ord (resource=" + _input + ")", pe); throw e; } }
public override int GetOrd(int docId) { if (docId < 0 || docId >= _outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (_outerInstance.MAX_DOC - 1) + "; got " + docId); } _input.Seek(_field.DataStartFilePointer + _field.NumValues * (9 + _field.Pattern.Length + _field.MaxLength) + docId * (1 + _field.OrdPattern.Length)); SimpleTextUtil.ReadLine(_input, _scratch); try { // LUCNENENET: .NET doesn't have a way to specify a pattern with integer, but all of the standard ones are built in. return(int.Parse(_scratch.Utf8ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture) - 1); } catch (Exception pe) { var e = new CorruptIndexException("failed to parse ord (resource=" + _input + ")", pe); throw e; } }
public override int GetOrd(int docID) { if (docID < 0 || docID >= outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (outerInstance.MAX_DOC - 1) + "; got " + docID); } @in.Seek(field.DataStartFilePointer + field.NumValues * (9 + field.Pattern.Length + field.MaxLength) + docID * (1 + field.OrdPattern.Length)); SimpleTextUtil.ReadLine(@in, scratch); try { return((long)(int)ordDecoder.Parse(scratch.Utf8ToString()) - 1); } catch (ParseException pe) { CorruptIndexException e = new CorruptIndexException("failed to parse ord (resource=" + @in + ")"); e.initCause(pe); throw e; } }
public SimpleTextDocValuesReader(SegmentReadState state, string ext) { DATA = state.Directory.OpenInput( IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, ext), state.Context); MAX_DOC = state.SegmentInfo.DocCount; while (true) { ReadLine(); if (SCRATCH.Equals(SimpleTextDocValuesWriter.END)) { break; } Debug.Assert(StartsWith(SimpleTextDocValuesWriter.FIELD), SCRATCH.Utf8ToString()); var fieldName = StripPrefix(SimpleTextDocValuesWriter.FIELD); var field = new OneField(); FIELDS[fieldName] = field; ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.TYPE), SCRATCH.Utf8ToString()); var dvType = (FieldInfo.DocValuesType_e) Enum.Parse(typeof(FieldInfo.DocValuesType_e), StripPrefix(SimpleTextDocValuesWriter.TYPE)); if (dvType == FieldInfo.DocValuesType_e.NUMERIC) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MINVALUE), "got " + SCRATCH.Utf8ToString() + " field=" + fieldName + " ext=" + ext); field.MinValue = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.MINVALUE)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (1 + field.Pattern.Length + 2) * MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.BINARY) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength + 2) * MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.SORTED || dvType == FieldInfo.DocValuesType_e.SORTED_SET) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.NUMVALUES)); field.NumValues = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.NUMVALUES)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.ORDPATTERN)); field.OrdPattern = StripPrefix(SimpleTextDocValuesWriter.ORDPATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength) * field.NumValues + (1 + field.OrdPattern.Length) * MAX_DOC); } else { throw new InvalidEnumArgumentException(); } } // We should only be called from above if at least one // field has DVs: Debug.Assert(FIELDS.Count > 0); }
public override int NextDoc() { bool first = true; _in.Seek(_nextDocStart); long posStart = 0; while (true) { long lineStart = _in.FilePointer; SimpleTextUtil.ReadLine(_in, _scratch); //System.out.println("NEXT DOC: " + scratch.utf8ToString()); if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.DOC)) { if (!first && (_liveDocs == null || _liveDocs.Get(_docId))) { _nextDocStart = lineStart; _in.Seek(posStart); return(_docId); } UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.DOC.Length, _scratch.Length - SimpleTextFieldsWriter.DOC.Length, _scratchUtf16); _docId = ArrayUtil.ParseInt(_scratchUtf16.Chars, 0, _scratchUtf16.Length); _tf = 0; first = false; } else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.FREQ)) { UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.FREQ.Length, _scratch.Length - SimpleTextFieldsWriter.FREQ.Length, _scratchUtf16); _tf = ArrayUtil.ParseInt(_scratchUtf16.Chars, 0, _scratchUtf16.Length); posStart = _in.FilePointer; } else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.POS)) { // skip } else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.START_OFFSET)) { // skip } else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.END_OFFSET)) { // skip } else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.PAYLOAD)) { // skip } else { Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.TERM) || StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.FIELD) || StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.END)); if (!first && (_liveDocs == null || _liveDocs.Get(_docId))) { _nextDocStart = lineStart; _in.Seek(posStart); return(_docId); } return(_docId = NO_MORE_DOCS); } } }
public SimpleTextDocValuesReader(SegmentReadState state, string ext) { DATA = state.Directory.OpenInput( IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, ext), state.Context); MAX_DOC = state.SegmentInfo.DocCount; while (true) { ReadLine(); if (SCRATCH.Equals(SimpleTextDocValuesWriter.END)) { break; } Debug.Assert(StartsWith(SimpleTextDocValuesWriter.FIELD), SCRATCH.Utf8ToString()); var fieldName = StripPrefix(SimpleTextDocValuesWriter.FIELD); var field = new OneField(); FIELDS[fieldName] = field; ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.TYPE), SCRATCH.Utf8ToString()); var dvType = (FieldInfo.DocValuesType_e) Enum.Parse(typeof (FieldInfo.DocValuesType_e), StripPrefix(SimpleTextDocValuesWriter.TYPE)); if (dvType == FieldInfo.DocValuesType_e.NUMERIC) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MINVALUE), "got " + SCRATCH.Utf8ToString() + " field=" + fieldName + " ext=" + ext); field.MinValue = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.MINVALUE)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (1 + field.Pattern.Length + 2)*MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.BINARY) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength + 2)*MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.SORTED || dvType == FieldInfo.DocValuesType_e.SORTED_SET) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.NUMVALUES)); field.NumValues = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.NUMVALUES)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.ORDPATTERN)); field.OrdPattern = StripPrefix(SimpleTextDocValuesWriter.ORDPATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength)*field.NumValues + (1 + field.OrdPattern.Length)*MAX_DOC); } else { throw new InvalidEnumArgumentException(); } } // We should only be called from above if at least one // field has DVs: Debug.Assert(FIELDS.Count > 0); }
public override void VisitDocument(int n, StoredFieldVisitor visitor) { _input.Seek(_offsets[n]); ReadLine(); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextStoredFieldsWriter.NUM)); var numFields = ParseIntAt(SimpleTextStoredFieldsWriter.NUM.Length); for (var i = 0; i < numFields; i++) { ReadLine(); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextStoredFieldsWriter.FIELD)); int fieldNumber = ParseIntAt(SimpleTextStoredFieldsWriter.FIELD.Length); FieldInfo fieldInfo = _fieldInfos.FieldInfo(fieldNumber); ReadLine(); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextStoredFieldsWriter.NAME)); ReadLine(); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextStoredFieldsWriter.TYPE)); BytesRef type; if (EqualsAt(SimpleTextStoredFieldsWriter.TYPE_STRING, _scratch, SimpleTextStoredFieldsWriter.TYPE.Length)) { type = SimpleTextStoredFieldsWriter.TYPE_STRING; } else if (EqualsAt(SimpleTextStoredFieldsWriter.TYPE_BINARY, _scratch, SimpleTextStoredFieldsWriter.TYPE.Length)) { type = SimpleTextStoredFieldsWriter.TYPE_BINARY; } else if (EqualsAt(SimpleTextStoredFieldsWriter.TYPE_INT, _scratch, SimpleTextStoredFieldsWriter.TYPE.Length)) { type = SimpleTextStoredFieldsWriter.TYPE_INT; } else if (EqualsAt(SimpleTextStoredFieldsWriter.TYPE_LONG, _scratch, SimpleTextStoredFieldsWriter.TYPE.Length)) { type = SimpleTextStoredFieldsWriter.TYPE_LONG; } else if (EqualsAt(SimpleTextStoredFieldsWriter.TYPE_FLOAT, _scratch, SimpleTextStoredFieldsWriter.TYPE.Length)) { type = SimpleTextStoredFieldsWriter.TYPE_FLOAT; } else if (EqualsAt(SimpleTextStoredFieldsWriter.TYPE_DOUBLE, _scratch, SimpleTextStoredFieldsWriter.TYPE.Length)) { type = SimpleTextStoredFieldsWriter.TYPE_DOUBLE; } else { throw new Exception("unknown field type"); } switch (visitor.NeedsField(fieldInfo)) { case StoredFieldVisitor.Status.YES: ReadField(type, fieldInfo, visitor); break; case StoredFieldVisitor.Status.NO: ReadLine(); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextStoredFieldsWriter.VALUE)); break; case StoredFieldVisitor.Status.STOP: return; } } }