public static void CheckFooter(ChecksumIndexInput input) { var scratch = new BytesRef(); var expectedChecksum = string.Format("{0:D}", input.Checksum); ReadLine(input, scratch); if (StringHelper.StartsWith(scratch, CHECKSUM) == false) { throw new CorruptIndexException("SimpleText failure: expected checksum line but got " + scratch.Utf8ToString() + " (resource=" + input + ")"); } var actualChecksum = (new BytesRef(scratch.Bytes, CHECKSUM.Length, scratch.Length - CHECKSUM.Length)).Utf8ToString(); if (!expectedChecksum.Equals(actualChecksum)) { throw new CorruptIndexException("SimpleText checksum failure: " + actualChecksum + " != " + expectedChecksum + " (resource=" + input + ")"); } if (input.Length() != input.FilePointer) { throw new CorruptIndexException( "Unexpected stuff at the end of file, please be careful with your text editor! (resource=" + input + ")"); } }
public override void LookupOrd(long ord, BytesRef result) { if (ord < 0 || ord >= _field.NumValues) { throw new IndexOutOfRangeException("ord must be 0 .. " + (_field.NumValues - 1) + "; got " + ord); } _input.Seek(_field.DataStartFilePointer + ord * (9 + _field.Pattern.Length + _field.MaxLength)); SimpleTextUtil.ReadLine(_input, _scratch); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextDocValuesWriter.LENGTH), "got " + _scratch.Utf8ToString() + " in=" + _input); int len; try { len = (int) _decoderFormat.parse(_scratch.Bytes.SubList( _scratch.Offset + SimpleTextDocValuesWriter.LENGTH.Length, _scratch.Length - SimpleTextDocValuesWriter.LENGTH.Length).ToString()); } catch (Exception pe) { var e = new CorruptIndexException("failed to parse int length (resource=" + _input + ")", pe); throw e; } result.Bytes = new byte[len]; result.Offset = 0; result.Length = len; _input.ReadBytes(result.Bytes, 0, len); }
public override long Get(int docId) { if (docId < 0 || docId >= _outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (_outerInstance.MAX_DOC - 1) + "; got " + docId); } _input.Seek(_field.DataStartFilePointer + (1 + _field.Pattern.Length + 2) * docId); SimpleTextUtil.ReadLine(_input, _scratch); long bd; try { bd = long.Parse(_scratch.Utf8ToString()); } catch (FormatException ex) { throw new CorruptIndexException("failed to parse long value (resource=" + _input + ")", ex); } SimpleTextUtil.ReadLine(_input, _scratch); // read the line telling us if its real or not return(_field.MinValue + bd); }
public override long Get(int docId) { if (docId < 0 || docId >= _outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (_outerInstance.MAX_DOC - 1) + "; got " + docId); } _input.Seek(_field.DataStartFilePointer + (1 + _field.Pattern.Length + 2) * docId); SimpleTextUtil.ReadLine(_input, _scratch); decimal bd; try { // LUCNENENET: .NET doesn't have a way to specify a pattern with decimal, but all of the standard ones are built in. bd = decimal.Parse(_scratch.Utf8ToString(), NumberStyles.Float, CultureInfo.InvariantCulture); } catch (FormatException ex) { throw new CorruptIndexException("failed to parse long value (resource=" + _input + ")", ex); } SimpleTextUtil.ReadLine(_input, _scratch); // read the line telling us if its real or not return((long)BigInteger.Add(new BigInteger(_field.MinValue), new BigInteger(bd))); }
public override void LookupOrd(long ord, BytesRef result) { if (ord < 0 || ord >= field.NumValues) { throw new IndexOutOfRangeException("ord must be 0 .. " + (field.NumValues - 1) + "; got " + ord); } @in.Seek(field.DataStartFilePointer + ord * (9 + field.Pattern.Length + field.MaxLength)); SimpleTextUtil.ReadLine(@in, scratch); Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextDocValuesWriter.LENGTH), "got " + scratch.Utf8ToString() + " in=" + @in); int len; try { len = (int) decoder.parse(scratch.Bytes.SubList( scratch.Offset + SimpleTextDocValuesWriter.LENGTH.Length, scratch.Length - SimpleTextDocValuesWriter.LENGTH.Length).ToString()); } catch (ParseException pe) { CorruptIndexException e = new CorruptIndexException("failed to parse int length (resource=" + @in + ")"); e.initCause(pe); throw e; } result.Bytes = new sbyte[len]; result.Offset = 0; result.Length = len; @in.ReadBytes(result.Bytes, 0, len); }
public override void LookupOrd(long ord, BytesRef result) { if (ord < 0 || ord >= _field.NumValues) { throw new IndexOutOfRangeException("ord must be 0 .. " + (_field.NumValues - 1) + "; got " + ord); } _input.Seek(_field.DataStartFilePointer + ord * (9 + _field.Pattern.Length + _field.MaxLength)); SimpleTextUtil.ReadLine(_input, _scratch); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextDocValuesWriter.LENGTH), "got " + _scratch.Utf8ToString() + " in=" + _input); int len; try { // LUCNENENET: .NET doesn't have a way to specify a pattern with integer, but all of the standard ones are built in. len = int.Parse(Encoding.UTF8.GetString(_scratch.Bytes, _scratch.Offset + SimpleTextDocValuesWriter.LENGTH.Length, _scratch.Length - SimpleTextDocValuesWriter.LENGTH.Length), NumberStyles.Integer, CultureInfo.InvariantCulture); } catch (Exception pe) { var e = new CorruptIndexException("failed to parse int length (resource=" + _input + ")", pe); throw e; } result.Bytes = new byte[len]; result.Offset = 0; result.Length = len; _input.ReadBytes(result.Bytes, 0, len); }
public override int GetOrd(int docId) { if (docId < 0 || docId >= _outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (_outerInstance.MAX_DOC - 1) + "; got " + docId); } _input.Seek(_field.DataStartFilePointer + _field.NumValues * (9 + _field.Pattern.Length + _field.MaxLength) + docId * (1 + _field.OrdPattern.Length)); SimpleTextUtil.ReadLine(_input, _scratch); try { return(_scratch.Utf8ToString().ToString(_ordDecoderFormat) - 1); } catch (Exception pe) { var e = new CorruptIndexException("failed to parse ord (resource=" + _input + ")", pe); throw e; } }
public override int GetOrd(int docId) { if (docId < 0 || docId >= _outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (_outerInstance.MAX_DOC - 1) + "; got " + docId); } _input.Seek(_field.DataStartFilePointer + _field.NumValues * (9 + _field.Pattern.Length + _field.MaxLength) + docId * (1 + _field.OrdPattern.Length)); SimpleTextUtil.ReadLine(_input, _scratch); try { // LUCNENENET: .NET doesn't have a way to specify a pattern with integer, but all of the standard ones are built in. return(int.Parse(_scratch.Utf8ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture) - 1); } catch (Exception pe) { var e = new CorruptIndexException("failed to parse ord (resource=" + _input + ")", pe); throw e; } }
public override int GetOrd(int docID) { if (docID < 0 || docID >= outerInstance.MAX_DOC) { throw new IndexOutOfRangeException("docID must be 0 .. " + (outerInstance.MAX_DOC - 1) + "; got " + docID); } @in.Seek(field.DataStartFilePointer + field.NumValues * (9 + field.Pattern.Length + field.MaxLength) + docID * (1 + field.OrdPattern.Length)); SimpleTextUtil.ReadLine(@in, scratch); try { return((long)(int)ordDecoder.Parse(scratch.Utf8ToString()) - 1); } catch (ParseException pe) { CorruptIndexException e = new CorruptIndexException("failed to parse ord (resource=" + @in + ")"); e.initCause(pe); throw e; } }
public override int NextPosition() { int pos; if (_readPositions) { SimpleTextUtil.ReadLine(_in, _scratch); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.POS), "got line=" + _scratch.Utf8ToString()); UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.POS.Length, _scratch.Length - SimpleTextFieldsWriter.POS.Length, _scratchUtf162); pos = ArrayUtil.ParseInt(_scratchUtf162.Chars, 0, _scratchUtf162.Length); } else { pos = -1; } if (_readOffsets) { SimpleTextUtil.ReadLine(_in, _scratch); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.START_OFFSET), "got line=" + _scratch.Utf8ToString()); UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.START_OFFSET.Length, _scratch.Length - SimpleTextFieldsWriter.START_OFFSET.Length, _scratchUtf162); _startOffset = ArrayUtil.ParseInt(_scratchUtf162.Chars, 0, _scratchUtf162.Length); SimpleTextUtil.ReadLine(_in, _scratch); Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.END_OFFSET), "got line=" + _scratch.Utf8ToString()); UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.END_OFFSET.Length, _scratch.Length - SimpleTextFieldsWriter.END_OFFSET.Length, _scratchUtf162); _endOffset = ArrayUtil.ParseInt(_scratchUtf162.Chars, 0, _scratchUtf162.Length); } long fp = _in.FilePointer; SimpleTextUtil.ReadLine(_in, _scratch); if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.PAYLOAD)) { int len = _scratch.Length - SimpleTextFieldsWriter.PAYLOAD.Length; if (_scratch2.Bytes.Length < len) { _scratch2.Grow(len); } Array.Copy(_scratch.Bytes, SimpleTextFieldsWriter.PAYLOAD.Length, _scratch2.Bytes, 0, len); _scratch2.Length = len; _payload = _scratch2; } else { _payload = null; _in.Seek(fp); } return(pos); }
public override int NextDoc() { if (_docId == NO_MORE_DOCS) { return(_docId); } bool first = true; int termFreq = 0; while (true) { long lineStart = _in.FilePointer; SimpleTextUtil.ReadLine(_in, _scratch); if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.DOC)) { if (!first && (_liveDocs == null || _liveDocs.Get(_docId))) { _in.Seek(lineStart); if (!_omitTf) { _tf = termFreq; } return(_docId); } UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.DOC.Length, _scratch.Length - SimpleTextFieldsWriter.DOC.Length, _scratchUtf16); _docId = ArrayUtil.ParseInt(_scratchUtf16.Chars, 0, _scratchUtf16.Length); termFreq = 0; first = false; } else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.FREQ)) { UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.FREQ.Length, _scratch.Length - SimpleTextFieldsWriter.FREQ.Length, _scratchUtf16); termFreq = ArrayUtil.ParseInt(_scratchUtf16.Chars, 0, _scratchUtf16.Length); } else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.POS)) { // skip termFreq++; } else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.START_OFFSET)) { // skip } else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.END_OFFSET)) { // skip } else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.PAYLOAD)) { // skip } else { Debug.Assert( StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.TERM) || StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.FIELD) || StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.END), "scratch=" + _scratch.Utf8ToString()); if (!first && (_liveDocs == null || _liveDocs.Get(_docId))) { _in.Seek(lineStart); if (!_omitTf) { _tf = termFreq; } return(_docId); } return(_docId = NO_MORE_DOCS); } } }