private IDictionary <string, long?> ReadFields(IndexInput @in) { ChecksumIndexInput input = new BufferedChecksumIndexInput(@in); var scratch = new BytesRef(10); // LUCENENET specific: Use StringComparer.Ordinal to get the same ordering as Java var fields = new JCG.SortedDictionary <string, long?>(StringComparer.Ordinal); while (true) { SimpleTextUtil.ReadLine(input, scratch); if (scratch.Equals(SimpleTextFieldsWriter.END)) { SimpleTextUtil.CheckFooter(input); return(fields); } if (StringHelper.StartsWith(scratch, SimpleTextFieldsWriter.FIELD)) { var fieldName = Encoding.UTF8.GetString(scratch.Bytes, scratch.Offset + SimpleTextFieldsWriter.FIELD.Length, scratch.Length - SimpleTextFieldsWriter.FIELD.Length); fields[fieldName] = input.GetFilePointer(); } } }
public override IBits ReadLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) { if (Debugging.AssertsEnabled) { Debugging.Assert(info.HasDeletions); } var scratch = new BytesRef(); var scratchUtf16 = new CharsRef(); var fileName = IndexFileNames.FileNameFromGeneration(info.Info.Name, LIVEDOCS_EXTENSION, info.DelGen); ChecksumIndexInput input = null; var success = false; try { input = dir.OpenChecksumInput(fileName, context); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SIZE)); } var size = ParseInt32At(scratch, SIZE.Length, scratchUtf16); var bits = new BitSet(size); SimpleTextUtil.ReadLine(input, scratch); while (!scratch.Equals(END)) { if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, DOC)); } var docid = ParseInt32At(scratch, DOC.Length, scratchUtf16); bits.Set(docid); SimpleTextUtil.ReadLine(input, scratch); } SimpleTextUtil.CheckFooter(input); success = true; return(new SimpleTextBits(bits, size)); } finally { if (success) { IOUtils.Dispose(input); } else { IOUtils.DisposeWhileHandlingException(input); } } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public void checkIntegrity() throws java.io.IOException public override void checkIntegrity() { BytesRef scratch = new BytesRef(); IndexInput clone = data.clone(); clone.seek(0); ChecksumIndexInput input = new BufferedChecksumIndexInput(clone); while (true) { SimpleTextUtil.ReadLine(input, scratch); if (scratch.Equals(END)) { SimpleTextUtil.CheckFooter(input); break; } } }
/// <remarks> /// we don't actually write a .fdx-like index, instead we read the /// stored fields file in entirety up-front and save the offsets /// so we can seek to the documents later. /// </remarks> private void ReadIndex(int size) { ChecksumIndexInput input = new BufferedChecksumIndexInput(_input); _offsets = new long[size]; var upto = 0; while (!_scratch.Equals(SimpleTextStoredFieldsWriter.END)) { SimpleTextUtil.ReadLine(input, _scratch); if (StringHelper.StartsWith(_scratch, SimpleTextStoredFieldsWriter.DOC)) { _offsets[upto] = input.GetFilePointer(); upto++; } } SimpleTextUtil.CheckFooter(input); Debug.Assert(upto == _offsets.Length); }
// we don't actually write a .tvx-like index, instead we read the // vectors file in entirety up-front and save the offsets // so we can seek to the data later. private void ReadIndex(int maxDoc) { ChecksumIndexInput input = new BufferedChecksumIndexInput(_input); _offsets = new long[maxDoc]; int upto = 0; while (!_scratch.Equals(END)) { SimpleTextUtil.ReadLine(input, _scratch); if (StringHelper.StartsWith(_scratch, DOC)) { _offsets[upto] = input.FilePointer; upto++; } } SimpleTextUtil.CheckFooter(input); Debug.Assert(upto == _offsets.Length); }
public override void CheckIntegrity() { var iScratch = new BytesRef(); var clone = (IndexInput)DATA.Clone(); clone.Seek(0); ChecksumIndexInput input = new BufferedChecksumIndexInput(clone); while (true) { SimpleTextUtil.ReadLine(input, iScratch); if (!iScratch.Equals(SimpleTextDocValuesWriter.END)) { continue; } SimpleTextUtil.CheckFooter(input); break; } }
// we don't actually write a .tvx-like index, instead we read the // vectors file in entirety up-front and save the offsets // so we can seek to the data later. private void ReadIndex(int maxDoc) { ChecksumIndexInput input = new BufferedChecksumIndexInput(_input); _offsets = new long[maxDoc]; int upto = 0; while (!_scratch.Equals(SimpleTextTermVectorsWriter.END)) { SimpleTextUtil.ReadLine(input, _scratch); if (StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.DOC)) { _offsets[upto] = input.Position; // LUCENENET specific: Renamed from getFilePointer() to match FileStream upto++; } } SimpleTextUtil.CheckFooter(input); if (Debugging.AssertsEnabled) { Debugging.Assert(upto == _offsets.Length); } }
private SortedDictionary <string, long?> ReadFields(IndexInput @in) { ChecksumIndexInput input = new BufferedChecksumIndexInput(@in); BytesRef scratch = new BytesRef(10); SortedDictionary <string, long?> fields = new SortedDictionary <string, long?>(); while (true) { SimpleTextUtil.ReadLine(input, scratch); if (scratch.Equals(END)) { SimpleTextUtil.CheckFooter(input); return(fields); } else if (StringHelper.StartsWith(scratch, FIELD)) { string fieldName = new string(scratch.Bytes, scratch.Offset + FIELD.length, scratch.Length - FIELD.length, StandardCharsets.UTF_8); fields[fieldName] = input.FilePointer; } } }
private SortedDictionary <string, long?> ReadFields(IndexInput @in) { ChecksumIndexInput input = new BufferedChecksumIndexInput(@in); var scratch = new BytesRef(10); var fields = new SortedDictionary <string, long?>(); while (true) { SimpleTextUtil.ReadLine(input, scratch); if (scratch.Equals(SimpleTextFieldsWriter.END)) { SimpleTextUtil.CheckFooter(input); return(fields); } if (StringHelper.StartsWith(scratch, SimpleTextFieldsWriter.FIELD)) { var fieldName = Encoding.UTF8.GetString(scratch.Bytes, scratch.Offset + SimpleTextFieldsWriter.FIELD.Length, scratch.Length - SimpleTextFieldsWriter.FIELD.Length); fields[fieldName] = input.FilePointer; } } }
public override SegmentInfo Read(Directory directory, string segmentName, IOContext context) { var scratch = new BytesRef(); string segFileName = IndexFileNames.SegmentFileName(segmentName, "", SimpleTextSegmentInfoFormat.SI_EXTENSION); ChecksumIndexInput input = directory.OpenChecksumInput(segFileName, context); bool success = false; try { SimpleTextUtil.ReadLine(input, scratch); Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextSegmentInfoWriter.SI_VERSION)); string version = ReadString(SimpleTextSegmentInfoWriter.SI_VERSION.Length, scratch); SimpleTextUtil.ReadLine(input, scratch); Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextSegmentInfoWriter.SI_DOCCOUNT)); int docCount = Convert.ToInt32(ReadString(SimpleTextSegmentInfoWriter.SI_DOCCOUNT.Length, scratch), CultureInfo.InvariantCulture); SimpleTextUtil.ReadLine(input, scratch); Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextSegmentInfoWriter.SI_USECOMPOUND)); bool isCompoundFile = Convert.ToBoolean(ReadString(SimpleTextSegmentInfoWriter.SI_USECOMPOUND.Length, scratch), CultureInfo.InvariantCulture); SimpleTextUtil.ReadLine(input, scratch); Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextSegmentInfoWriter.SI_NUM_DIAG)); int numDiag = Convert.ToInt32(ReadString(SimpleTextSegmentInfoWriter.SI_NUM_DIAG.Length, scratch), CultureInfo.InvariantCulture); IDictionary <string, string> diagnostics = new Dictionary <string, string>(); for (int i = 0; i < numDiag; i++) { SimpleTextUtil.ReadLine(input, scratch); Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextSegmentInfoWriter.SI_DIAG_KEY)); string key = ReadString(SimpleTextSegmentInfoWriter.SI_DIAG_KEY.Length, scratch); SimpleTextUtil.ReadLine(input, scratch); Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextSegmentInfoWriter.SI_DIAG_VALUE)); string value = ReadString(SimpleTextSegmentInfoWriter.SI_DIAG_VALUE.Length, scratch); diagnostics[key] = value; } SimpleTextUtil.ReadLine(input, scratch); Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextSegmentInfoWriter.SI_NUM_FILES)); int numFiles = Convert.ToInt32(ReadString(SimpleTextSegmentInfoWriter.SI_NUM_FILES.Length, scratch), CultureInfo.InvariantCulture); var files = new HashSet <string>(); for (int i = 0; i < numFiles; i++) { SimpleTextUtil.ReadLine(input, scratch); Debug.Assert(StringHelper.StartsWith(scratch, SimpleTextSegmentInfoWriter.SI_FILE)); string fileName = ReadString(SimpleTextSegmentInfoWriter.SI_FILE.Length, scratch); files.Add(fileName); } SimpleTextUtil.CheckFooter(input); var info = new SegmentInfo(directory, version, segmentName, docCount, isCompoundFile, null, diagnostics); info.SetFiles(files); success = true; return(info); } finally { if (!success) { IOUtils.DisposeWhileHandlingException(input); } else { input.Dispose(); } } }
public override FieldInfos Read(Directory directory, string segmentName, string segmentSuffix, IOContext iocontext) { var fileName = IndexFileNames.SegmentFileName(segmentName, segmentSuffix, SimpleTextFieldInfosWriter.FIELD_INFOS_EXTENSION); var input = directory.OpenChecksumInput(fileName, iocontext); var scratch = new BytesRef(); var success = false; try { SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.NUMFIELDS)); } var size = Convert.ToInt32(ReadString(SimpleTextFieldInfosWriter.NUMFIELDS.Length, scratch), CultureInfo.InvariantCulture); var infos = new FieldInfo[size]; for (var i = 0; i < size; i++) { SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.NAME)); } string name = ReadString(SimpleTextFieldInfosWriter.NAME.Length, scratch); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.NUMBER)); } int fieldNumber = Convert.ToInt32(ReadString(SimpleTextFieldInfosWriter.NUMBER.Length, scratch), CultureInfo.InvariantCulture); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.ISINDEXED)); } bool isIndexed = Convert.ToBoolean(ReadString(SimpleTextFieldInfosWriter.ISINDEXED.Length, scratch), CultureInfo.InvariantCulture); IndexOptions indexOptions; if (isIndexed) { SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.INDEXOPTIONS)); } indexOptions = (IndexOptions)Enum.Parse(typeof(IndexOptions), ReadString(SimpleTextFieldInfosWriter.INDEXOPTIONS.Length, scratch)); } else { indexOptions = IndexOptions.NONE; } SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.STORETV)); } bool storeTermVector = Convert.ToBoolean(ReadString(SimpleTextFieldInfosWriter.STORETV.Length, scratch), CultureInfo.InvariantCulture); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.PAYLOADS)); } bool storePayloads = Convert.ToBoolean(ReadString(SimpleTextFieldInfosWriter.PAYLOADS.Length, scratch), CultureInfo.InvariantCulture); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.NORMS)); } bool omitNorms = !Convert.ToBoolean(ReadString(SimpleTextFieldInfosWriter.NORMS.Length, scratch), CultureInfo.InvariantCulture); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.NORMS_TYPE)); } string nrmType = ReadString(SimpleTextFieldInfosWriter.NORMS_TYPE.Length, scratch); Index.DocValuesType normsType = DocValuesType(nrmType); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.DOCVALUES)); } string dvType = ReadString(SimpleTextFieldInfosWriter.DOCVALUES.Length, scratch); Index.DocValuesType docValuesType = DocValuesType(dvType); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.DOCVALUES_GEN)); } long dvGen = Convert.ToInt64(ReadString(SimpleTextFieldInfosWriter.DOCVALUES_GEN.Length, scratch), CultureInfo.InvariantCulture); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.NUM_ATTS)); } int numAtts = Convert.ToInt32(ReadString(SimpleTextFieldInfosWriter.NUM_ATTS.Length, scratch), CultureInfo.InvariantCulture); IDictionary <string, string> atts = new Dictionary <string, string>(); for (int j = 0; j < numAtts; j++) { SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.ATT_KEY)); } string key = ReadString(SimpleTextFieldInfosWriter.ATT_KEY.Length, scratch); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SimpleTextFieldInfosWriter.ATT_VALUE)); } string value = ReadString(SimpleTextFieldInfosWriter.ATT_VALUE.Length, scratch); atts[key] = value; } infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, normsType, new ReadOnlyDictionary <string, string>(atts)) { DocValuesGen = dvGen }; } SimpleTextUtil.CheckFooter(input); var fieldInfos = new FieldInfos(infos); success = true; return(fieldInfos); } finally { if (success) { input.Dispose(); } else { IOUtils.DisposeWhileHandlingException(input); } } }