public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) { string termsFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, FSTTermsWriter.TERMS_EXTENSION); this.postingsReader = postingsReader; IndexInput @in = state.Directory.OpenInput(termsFileName, state.Context); bool success = false; try { version = ReadHeader(@in); if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) { CodecUtil.ChecksumEntireFile(@in); } this.postingsReader.Init(@in); SeekDir(@in); FieldInfos fieldInfos = state.FieldInfos; int numFields = @in.ReadVInt(); for (int i = 0; i < numFields; i++) { int fieldNumber = @in.ReadVInt(); FieldInfo fieldInfo = fieldInfos.FieldInfo(fieldNumber); long numTerms = @in.ReadVLong(); long sumTotalTermFreq = fieldInfo.FieldIndexOptions == IndexOptions.DOCS_ONLY ? -1 : @in.ReadVLong(); long sumDocFreq = @in.ReadVLong(); int docCount = @in.ReadVInt(); int longsSize = @in.ReadVInt(); TermsReader current = new TermsReader(this, fieldInfo, @in, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize); TermsReader previous = fields[fieldInfo.Name] = current; CheckFieldSummary(state.SegmentInfo, @in, current, previous); } success = true; } finally { if (success) { IOUtils.Close(@in); } else { IOUtils.CloseWhileHandlingException(@in); } } }
public override FieldsProducer FieldsProducer(SegmentReadState state) { PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.Directory, state.FieldInfos, state.SegmentInfo, state.Context, state.SegmentSuffix); bool success = false; try { FieldsProducer ret = new FSTTermsReader(state, postingsReader); success = true; return ret; } finally { if (!success) { IOUtils.CloseWhileHandlingException(postingsReader); } } }
public override FieldsProducer FieldsProducer(SegmentReadState state) { PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.Directory, state.FieldInfos, state.SegmentInfo, state.Context, state.SegmentSuffix); bool success = false; try { FieldsProducer ret = new FSTTermsReader(state, postingsReader); success = true; return(ret); } finally { if (!success) { IOUtils.DisposeWhileHandlingException(postingsReader); } } }
public override FieldsProducer FieldsProducer(SegmentReadState state) { PostingsReaderBase docsReader = null; PostingsReaderBase pulsingReader = null; bool success = false; try { docsReader = _wrappedPostingsBaseFormat.PostingsReaderBase(state); pulsingReader = new PulsingPostingsReader(state, docsReader); FieldsProducer ret = new FSTOrdTermsReader(state, pulsingReader); success = true; return ret; } finally { if (!success) { IOUtils.CloseWhileHandlingException(docsReader, pulsingReader); } } }
public SimpleTextFieldsReader(SegmentReadState state) { _maxDoc = state.SegmentInfo.DocCount; _fieldInfos = state.FieldInfos; _input = state.Directory.OpenInput( SimpleTextPostingsFormat.GetPostingsFileName(state.SegmentInfo.Name, state.SegmentSuffix), state.Context); bool success = false; try { _fields = ReadFields((IndexInput)_input.Clone()); success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(); } } }
public override FieldsProducer FieldsProducer(SegmentReadState state) { PostingsReaderBase docsReader = null; PostingsReaderBase pulsingReader = null; bool success = false; try { docsReader = _wrappedPostingsBaseFormat.PostingsReaderBase(state); pulsingReader = new PulsingPostingsReader(state, docsReader); FieldsProducer ret = new FSTOrdTermsReader(state, pulsingReader); success = true; return(ret); } finally { if (!success) { IOUtils.CloseWhileHandlingException(docsReader, pulsingReader); } } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public DirectFields(index.SegmentReadState state, index.Fields fields, int minSkipCount, int lowFreqCutoff) throws java.io.IOException public DirectFields(SegmentReadState state, Fields fields, int minSkipCount, int lowFreqCutoff) { foreach (string field in fields) { this.fields[field] = new DirectField(state, field, fields.terms(field), minSkipCount, lowFreqCutoff); } }
public override FieldsProducer FieldsProducer(SegmentReadState state) { return new SimpleTextFieldsReader(state); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public DirectField(index.SegmentReadState state, String field, index.Terms termsIn, int minSkipCount, int lowFreqCutoff) throws java.io.IOException public DirectField(SegmentReadState state, string field, Terms termsIn, int minSkipCount, int lowFreqCutoff) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final index.FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field); FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field); sumTotalTermFreq = termsIn.SumTotalTermFreq; sumDocFreq = termsIn.SumDocFreq; docCount = termsIn.DocCount; //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int numTerms = (int) termsIn.size(); int numTerms = (int) termsIn.size(); if (numTerms == -1) { throw new System.ArgumentException("codec does not provide Terms.size()"); } terms = new TermAndSkip[numTerms]; termOffsets = new int[1 + numTerms]; sbyte[] termBytes = new sbyte[1024]; this.minSkipCount = minSkipCount; hasFreq = fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_ONLY) > 0; hasPos = fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) > 0; hasOffsets_Renamed = fieldInfo.IndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) > 0; hasPayloads_Renamed = fieldInfo.hasPayloads(); BytesRef term; DocsEnum docsEnum = null; DocsAndPositionsEnum docsAndPositionsEnum = null; //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final index.TermsEnum termsEnum = termsIn.iterator(null); TermsEnum termsEnum = termsIn.iterator(null); int termOffset = 0; //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final IntArrayWriter scratch = new IntArrayWriter(); IntArrayWriter scratch = new IntArrayWriter(); // Used for payloads, if any: //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final store.RAMOutputStream ros = new store.RAMOutputStream(); RAMOutputStream ros = new RAMOutputStream(); // if (DEBUG) { // System.out.println("\nLOAD terms seg=" + state.segmentInfo.name + " field=" + field + " hasOffsets=" + hasOffsets + " hasFreq=" + hasFreq + " hasPos=" + hasPos + " hasPayloads=" + hasPayloads); // } while ((term = termsEnum.next()) != null) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int docFreq = termsEnum.docFreq(); int docFreq = termsEnum.docFreq(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final long totalTermFreq = termsEnum.totalTermFreq(); long totalTermFreq = termsEnum.totalTermFreq(); // if (DEBUG) { // System.out.println(" term=" + term.utf8ToString()); // } termOffsets[count] = termOffset; if (termBytes.Length < (termOffset + term.length)) { termBytes = ArrayUtil.grow(termBytes, termOffset + term.length); } Array.Copy(term.bytes, term.offset, termBytes, termOffset, term.length); termOffset += term.length; termOffsets[count + 1] = termOffset; if (hasPos) { docsAndPositionsEnum = termsEnum.docsAndPositions(null, docsAndPositionsEnum); } else { docsEnum = termsEnum.docs(null, docsEnum); } //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final TermAndSkip ent; TermAndSkip ent; //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final index.DocsEnum docsEnum2; DocsEnum docsEnum2; if (hasPos) { docsEnum2 = docsAndPositionsEnum; } else { docsEnum2 = docsEnum; } int docID; if (docFreq <= lowFreqCutoff) { ros.reset(); // Pack postings for low-freq terms into a single int[]: while ((docID = docsEnum2.nextDoc()) != DocsEnum.NO_MORE_DOCS) { scratch.add(docID); if (hasFreq) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int freq = docsEnum2.freq(); int freq = docsEnum2.freq(); scratch.add(freq); if (hasPos) { for (int pos = 0; pos < freq; pos++) { scratch.add(docsAndPositionsEnum.nextPosition()); if (hasOffsets_Renamed) { scratch.add(docsAndPositionsEnum.startOffset()); scratch.add(docsAndPositionsEnum.endOffset()); } if (hasPayloads_Renamed) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final util.BytesRef payload = docsAndPositionsEnum.getPayload(); BytesRef payload = docsAndPositionsEnum.Payload; if (payload != null) { scratch.add(payload.length); ros.writeBytes(payload.bytes, payload.offset, payload.length); } else { scratch.add(0); } } } } } } //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final byte[] payloads; sbyte[] payloads; if (hasPayloads_Renamed) { ros.flush(); payloads = new sbyte[(int) ros.length()]; ros.writeTo(payloads, 0); } else { payloads = null; } //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int[] postings = scratch.get(); int[] postings = scratch.get(); ent = new LowFreqTerm(postings, payloads, docFreq, (int) totalTermFreq); } else { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int[] docs = new int[docFreq]; int[] docs = new int[docFreq]; //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int[] freqs; int[] freqs; //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int[][] positions; int[][] positions; //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final byte[][][] payloads; sbyte[][][] payloads; if (hasFreq) { freqs = new int[docFreq]; if (hasPos) { positions = new int[docFreq][]; if (hasPayloads_Renamed) { payloads = new sbyte[docFreq][][]; } else { payloads = null; } } else { positions = null; payloads = null; } } else { freqs = null; positions = null; payloads = null; } // Use separate int[] for the postings for high-freq // terms: int upto = 0; while ((docID = docsEnum2.nextDoc()) != DocsEnum.NO_MORE_DOCS) { docs[upto] = docID; if (hasFreq) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int freq = docsEnum2.freq(); int freq = docsEnum2.freq(); freqs[upto] = freq; if (hasPos) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int mult; int mult; if (hasOffsets_Renamed) { mult = 3; } else { mult = 1; } if (hasPayloads_Renamed) { payloads[upto] = new sbyte[freq][]; } positions[upto] = new int[mult*freq]; int posUpto = 0; for (int pos = 0; pos < freq; pos++) { positions[upto][posUpto] = docsAndPositionsEnum.nextPosition(); if (hasPayloads_Renamed) { BytesRef payload = docsAndPositionsEnum.Payload; if (payload != null) { sbyte[] payloadBytes = new sbyte[payload.length]; Array.Copy(payload.bytes, payload.offset, payloadBytes, 0, payload.length); payloads[upto][pos] = payloadBytes; } } posUpto++; if (hasOffsets_Renamed) { positions[upto][posUpto++] = docsAndPositionsEnum.startOffset(); positions[upto][posUpto++] = docsAndPositionsEnum.endOffset(); } } } } upto++; } Debug.Assert(upto == docFreq); ent = new HighFreqTerm(docs, freqs, positions, payloads, totalTermFreq); } terms[count] = ent; setSkips(count, termBytes); count++; } // End sentinel: termOffsets[count] = termOffset; finishSkips(); //System.out.println(skipCount + " skips: " + field); this.termBytes = new sbyte[termOffset]; Array.Copy(termBytes, 0, this.termBytes, 0, termOffset); // Pack skips: this.skips = new int[skipCount]; this.skipOffsets = new int[1 + numTerms]; int skipOffset = 0; for (int i = 0; i < numTerms; i++) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int[] termSkips = terms[i].skips; int[] termSkips = terms[i].skips; skipOffsets[i] = skipOffset; if (termSkips != null) { Array.Copy(termSkips, 0, skips, skipOffset, termSkips.Length); skipOffset += termSkips.Length; terms[i].skips = null; } } this.skipOffsets[numTerms] = skipOffset; Debug.Assert(skipOffset == skipCount); }
public override DocValuesProducer FieldsProducer(SegmentReadState state) { return(new SimpleTextDocValuesReader(state, "dat")); }
public SimpleTextNormsProducer(SegmentReadState state) : base(state, NORMS_SEG_EXTENSION) { // All we do is change the extension from .dat -> .len; // otherwise this is a normal simple doc values file: }
public override DocValuesProducer NormsProducer(SegmentReadState state) { return new SimpleTextNormsProducer(state); }
public override DocValuesProducer FieldsProducer(SegmentReadState state) { return new SimpleTextDocValuesReader(state, "dat"); }
public SimpleTextDocValuesReader(SegmentReadState state, string ext) { DATA = state.Directory.OpenInput( IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, ext), state.Context); MAX_DOC = state.SegmentInfo.DocCount; while (true) { ReadLine(); if (SCRATCH.Equals(SimpleTextDocValuesWriter.END)) { break; } Debug.Assert(StartsWith(SimpleTextDocValuesWriter.FIELD), SCRATCH.Utf8ToString()); var fieldName = StripPrefix(SimpleTextDocValuesWriter.FIELD); var field = new OneField(); FIELDS[fieldName] = field; ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.TYPE), SCRATCH.Utf8ToString()); var dvType = (FieldInfo.DocValuesType_e) Enum.Parse(typeof(FieldInfo.DocValuesType_e), StripPrefix(SimpleTextDocValuesWriter.TYPE)); if (dvType == FieldInfo.DocValuesType_e.NUMERIC) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MINVALUE), "got " + SCRATCH.Utf8ToString() + " field=" + fieldName + " ext=" + ext); field.MinValue = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.MINVALUE)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (1 + field.Pattern.Length + 2) * MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.BINARY) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength + 2) * MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.SORTED || dvType == FieldInfo.DocValuesType_e.SORTED_SET) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.NUMVALUES)); field.NumValues = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.NUMVALUES)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.ORDPATTERN)); field.OrdPattern = StripPrefix(SimpleTextDocValuesWriter.ORDPATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength) * field.NumValues + (1 + field.OrdPattern.Length) * MAX_DOC); } else { throw new InvalidEnumArgumentException(); } } // We should only be called from above if at least one // field has DVs: Debug.Assert(FIELDS.Count > 0); }
public override DocValuesProducer NormsProducer(SegmentReadState state) { return(new SimpleTextNormsProducer(state)); }
public SimpleTextDocValuesReader(SegmentReadState state, string ext) { DATA = state.Directory.OpenInput( IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, ext), state.Context); MAX_DOC = state.SegmentInfo.DocCount; while (true) { ReadLine(); if (SCRATCH.Equals(SimpleTextDocValuesWriter.END)) { break; } Debug.Assert(StartsWith(SimpleTextDocValuesWriter.FIELD), SCRATCH.Utf8ToString()); var fieldName = StripPrefix(SimpleTextDocValuesWriter.FIELD); var field = new OneField(); FIELDS[fieldName] = field; ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.TYPE), SCRATCH.Utf8ToString()); var dvType = (FieldInfo.DocValuesType_e) Enum.Parse(typeof (FieldInfo.DocValuesType_e), StripPrefix(SimpleTextDocValuesWriter.TYPE)); if (dvType == FieldInfo.DocValuesType_e.NUMERIC) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MINVALUE), "got " + SCRATCH.Utf8ToString() + " field=" + fieldName + " ext=" + ext); field.MinValue = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.MINVALUE)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (1 + field.Pattern.Length + 2)*MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.BINARY) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength + 2)*MAX_DOC); } else if (dvType == FieldInfo.DocValuesType_e.SORTED || dvType == FieldInfo.DocValuesType_e.SORTED_SET) { ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.NUMVALUES)); field.NumValues = Convert.ToInt64(StripPrefix(SimpleTextDocValuesWriter.NUMVALUES)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.MAXLENGTH)); field.MaxLength = Convert.ToInt32(StripPrefix(SimpleTextDocValuesWriter.MAXLENGTH)); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.PATTERN)); field.Pattern = StripPrefix(SimpleTextDocValuesWriter.PATTERN); ReadLine(); Debug.Assert(StartsWith(SimpleTextDocValuesWriter.ORDPATTERN)); field.OrdPattern = StripPrefix(SimpleTextDocValuesWriter.ORDPATTERN); field.DataStartFilePointer = DATA.FilePointer; DATA.Seek(DATA.FilePointer + (9 + field.Pattern.Length + field.MaxLength)*field.NumValues + (1 + field.OrdPattern.Length)*MAX_DOC); } else { throw new InvalidEnumArgumentException(); } } // We should only be called from above if at least one // field has DVs: Debug.Assert(FIELDS.Count > 0); }
public override FieldsProducer FieldsProducer(SegmentReadState state) { return(new SimpleTextFieldsReader(state)); }
public override DocValuesProducer FieldsProducer(SegmentReadState state) { return(new DirectDocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION)); }
public override FieldsProducer FieldsProducer(SegmentReadState state) { var postings = ForName("Lucene41").FieldsProducer(state); if (state.Context.Context != IOContext.Context.MERGE) { FieldsProducer loadedPostings; try { postings.CheckIntegrity(); loadedPostings = new DirectFields(state, postings, _minSkipCount, _lowFreqCutoff); } finally { postings.Dispose(); } return loadedPostings; } else { // Don't load postings for merge: return postings; } }
public override DocValuesProducer FieldsProducer(SegmentReadState state) { return new DirectDocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION); }