//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: public void test() throws Exception public virtual void test() { for (int i = 0; i < tests.Length; i += 3) { File f = new File(DICTIONARY_HOME, tests[i]); Debug.Assert(f.exists()); using (ZipFile zip = new ZipFile(f, StandardCharsets.UTF_8)) { ZipEntry dicEntry = zip.getEntry(tests[i + 1]); Debug.Assert(dicEntry != null); ZipEntry affEntry = zip.getEntry(tests[i + 2]); Debug.Assert(affEntry != null); using (System.IO.Stream dictionary = zip.getInputStream(dicEntry), System.IO.Stream affix = zip.getInputStream(affEntry)) { Dictionary dic = new Dictionary(affix, dictionary); Console.WriteLine(tests[i] + "\t" + RamUsageEstimator.humanSizeOf(dic) + "\t(" + "words=" + RamUsageEstimator.humanSizeOf(dic.words) + ", " + "flags=" + RamUsageEstimator.humanSizeOf(dic.flagLookup) + ", " + "strips=" + RamUsageEstimator.humanSizeOf(dic.stripData) + ", " + "conditions=" + RamUsageEstimator.humanSizeOf(dic.patterns) + ", " + "affixData=" + RamUsageEstimator.humanSizeOf(dic.affixData) + ", " + "prefixes=" + RamUsageEstimator.humanSizeOf(dic.prefixes) + ", " + "suffixes=" + RamUsageEstimator.humanSizeOf(dic.suffixes) + ")"); } } } }
/// <summary> /// Returns a String representation of the index data for debugging purposes. /// </summary> /// <returns> the string representation </returns> public override string ToString() { StringBuilder result = new StringBuilder(256); SortFields(); int sumPositions = 0; int sumTerms = 0; BytesRef spare = new BytesRef(); for (int i = 0; i < sortedFields.Length; i++) { KeyValuePair <string, Info> entry = sortedFields[i]; string fieldName = entry.Key; Info info = entry.Value; info.SortTerms(); result.Append(fieldName + ":\n"); SliceByteStartArray sliceArray = info.sliceArray; int numPositions = 0; Int32BlockPool.SliceReader postingsReader = new Int32BlockPool.SliceReader(intBlockPool); for (int j = 0; j < info.terms.Count; j++) { int ord = info.sortedTerms[j]; info.terms.Get(ord, spare); int freq = sliceArray.freq[ord]; result.Append("\t'" + spare + "':" + freq + ":"); postingsReader.Reset(sliceArray.start[ord], sliceArray.end[ord]); result.Append(" ["); int iters = storeOffsets ? 3 : 1; while (!postingsReader.IsEndOfSlice) { result.Append("("); for (int k = 0; k < iters; k++) { result.Append(postingsReader.ReadInt32()); if (k < iters - 1) { result.Append(", "); } } result.Append(")"); if (!postingsReader.IsEndOfSlice) { result.Append(","); } } result.Append("]"); result.Append("\n"); numPositions += freq; } result.Append("\tterms=" + info.terms.Count); result.Append(", positions=" + numPositions); result.Append(", memory=" + RamUsageEstimator.HumanReadableUnits(RamUsageEstimator.SizeOf(info))); result.Append("\n"); sumPositions += numPositions; sumTerms += info.terms.Count; } result.Append("\nfields=" + sortedFields.Length); result.Append(", terms=" + sumTerms); result.Append(", positions=" + sumPositions); result.Append(", memory=" + RamUsageEstimator.HumanReadableUnits(GetMemorySize())); return(result.ToString()); }
/// <summary> /// Return the number of bytes used by this instance. </summary> public virtual long RamBytesUsed() { // TODO: this is called per-doc-per-norms/dv-field, can we optimize this? long bytesUsed = RamUsageEstimator.AlignObjectSize(BaseRamBytesUsed()) + (pending != null ? RamUsageEstimator.SizeOf(pending) : 0L) + RamUsageEstimator.AlignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)RamUsageEstimator.NUM_BYTES_OBJECT_REF * values.Length); // values return(bytesUsed + valuesBytes); }
private long DocsWithFieldBytesUsed() { // size of the long[] + some overhead return(RamUsageEstimator.SizeOf(docsWithField.Bits) + 64); }
/// <summary> /// Returns a reasonable approximation of the main memory [bytes] consumed by /// this instance. Useful for smart memory sensititive caches/pools. </summary> /// <returns> the main memory consumption </returns> public virtual long GetMemorySize() { return(RamUsageEstimator.SizeOf(this)); }
/// <summary> Computes (and stores) the estimated size of the cache Value </summary> /// <seealso cref="EstimatedSize"> /// </seealso> public virtual void EstimateSize(RamUsageEstimator ramCalc) { long size = ramCalc.EstimateRamUsage(Value); EstimatedSize = RamUsageEstimator.HumanReadableUnits(size, new System.Globalization.NumberFormatInfo()); // {{Aroush-2.9}} in Java, the formater is set to "0.#", so we need to do the same in C# }
public override long RamBytesUsed() { return(base.RamBytesUsed() + RamUsageEstimator.SizeOf(averages) + RamUsageEstimator.SizeOf(minValues)); }
/// <summary> /// Computes (and stores) the estimated size of the cache <see cref="Value"/> /// </summary> /// <seealso cref="EstimatedSize"/> public void EstimateSize() { long bytesUsed = RamUsageEstimator.SizeOf(Value); size = RamUsageEstimator.HumanReadableUnits(bytesUsed); }
public override long RamBytesUsed() { return(RamUsageEstimator.AlignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 3 * RamUsageEstimator.NUM_BYTES_INT + RamUsageEstimator.NUM_BYTES_LONG + RamUsageEstimator.NUM_BYTES_OBJECT_REF) + RamUsageEstimator.SizeOf(Blocks)); // blocks ref - maskRight - bpvMinusBlockSize,valueCount,bitsPerValue }
public virtual long RamBytesUsed() { return(RamUsageEstimator.SizeOf(_filter.GetBits())); }
/// <summary> /// Loads the segment information at segment load time. /// </summary> /// <param name="indexEnum"> /// the term enum. </param> /// <param name="indexDivisor"> /// the index divisor. </param> /// <param name="tiiFileLength"> /// the size of the tii file, used to approximate the size of the /// buffer. </param> /// <param name="totalIndexInterval"> /// the total index interval. </param> public TermInfosReaderIndex(SegmentTermEnum indexEnum, int indexDivisor, long tiiFileLength, int totalIndexInterval) { this.TotalIndexInterval = totalIndexInterval; IndexSize = 1 + ((int)indexEnum.Size - 1) / indexDivisor; SkipInterval = indexEnum.SkipInterval; // this is only an inital size, it will be GCed once the build is complete long initialSize = (long)(tiiFileLength * 1.5) / indexDivisor; PagedBytes dataPagedBytes = new PagedBytes(EstimatePageBits(initialSize)); PagedBytesDataOutput dataOutput = dataPagedBytes.DataOutput; int bitEstimate = 1 + MathUtil.Log(tiiFileLength, 2); GrowableWriter indexToTerms = new GrowableWriter(bitEstimate, IndexSize, PackedInts.DEFAULT); string currentField = null; IList <string> fieldStrs = new List <string>(); int fieldCounter = -1; for (int i = 0; indexEnum.Next(); i++) { Term term = indexEnum.Term(); if (currentField == null || !currentField.Equals(term.Field)) { currentField = term.Field; fieldStrs.Add(currentField); fieldCounter++; } TermInfo termInfo = indexEnum.TermInfo(); indexToTerms.Set(i, dataOutput.Position); dataOutput.WriteVInt(fieldCounter); dataOutput.WriteString(term.Text()); dataOutput.WriteVInt(termInfo.DocFreq); if (termInfo.DocFreq >= SkipInterval) { dataOutput.WriteVInt(termInfo.SkipOffset); } dataOutput.WriteVLong(termInfo.FreqPointer); dataOutput.WriteVLong(termInfo.ProxPointer); dataOutput.WriteVLong(indexEnum.IndexPointer); for (int j = 1; j < indexDivisor; j++) { if (!indexEnum.Next()) { break; } } } Fields = new Term[fieldStrs.Count]; for (int i = 0; i < Fields.Length; i++) { Fields[i] = new Term(fieldStrs[i]); } dataPagedBytes.Freeze(true); DataInput = dataPagedBytes.DataInput; IndexToDataOffset = indexToTerms.Mutable; RamBytesUsed_Renamed = Fields.Length * (RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.ShallowSizeOfInstance(typeof(Term))) + dataPagedBytes.RamBytesUsed() + IndexToDataOffset.RamBytesUsed(); }
internal Lucene42DocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension) { MaxDoc = state.SegmentInfo.DocCount; string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension); // read in the entries from the metadata file. ChecksumIndexInput @in = state.Directory.OpenChecksumInput(metaName, state.Context); bool success = false; RamBytesUsed_Renamed = new AtomicLong(RamUsageEstimator.ShallowSizeOfInstance(this.GetType())); try { Version = CodecUtil.CheckHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT); Numerics = new Dictionary <int, NumericEntry>(); Binaries = new Dictionary <int, BinaryEntry>(); Fsts = new Dictionary <int, FSTEntry>(); ReadFields(@in, state.FieldInfos); if (Version >= VERSION_CHECKSUM) { CodecUtil.CheckFooter(@in); } else { CodecUtil.CheckEOF(@in); } success = true; } finally { if (success) { IOUtils.Close(@in); } else { IOUtils.CloseWhileHandlingException(@in); } } success = false; try { string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension); Data = state.Directory.OpenInput(dataName, state.Context); int version2 = CodecUtil.CheckHeader(Data, dataCodec, VERSION_START, VERSION_CURRENT); if (Version != version2) { throw new CorruptIndexException("Format versions mismatch"); } success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(this.Data); } } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: MemoryDocValuesProducer(index.SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws java.io.IOException internal MemoryDocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension) { maxDoc = state.segmentInfo.DocCount; string metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); // read in the entries from the metadata file. ChecksumIndexInput @in = state.directory.openChecksumInput(metaName, state.context); bool success = false; try { version = CodecUtil.checkHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT); numerics = new Dictionary <>(); binaries = new Dictionary <>(); fsts = new Dictionary <>(); readFields(@in, state.fieldInfos); if (version >= VERSION_CHECKSUM) { CodecUtil.checkFooter(@in); } else { CodecUtil.checkEOF(@in); } ramBytesUsed_Renamed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(this.GetType())); success = true; } finally { if (success) { IOUtils.close(@in); } else { IOUtils.closeWhileHandlingException(@in); } } success = false; try { string dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); data = state.directory.openInput(dataName, state.context); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int version2 = codecs.CodecUtil.checkHeader(data, dataCodec, VERSION_START, VERSION_CURRENT); int version2 = CodecUtil.checkHeader(data, dataCodec, VERSION_START, VERSION_CURRENT); if (version != version2) { throw new CorruptIndexException("Format versions mismatch"); } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(this.data); } } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: private index.NumericDocValues loadNumeric(index.FieldInfo field) throws java.io.IOException private NumericDocValues loadNumeric(FieldInfo field) { NumericEntry entry = numerics[field.number]; data.seek(entry.offset + entry.missingBytes); switch (entry.format) { case TABLE_COMPRESSED: int size = data.readVInt(); if (size > 256) { throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values, input=" + data); } //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final long decode[] = new long[size]; long[] decode = new long[size]; for (int i = 0; i < decode.Length; i++) { decode[i] = data.readLong(); } //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int formatID = data.readVInt(); int formatID = data.readVInt(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int bitsPerValue = data.readVInt(); int bitsPerValue = data.readVInt(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final util.packed.PackedInts.Reader ordsReader = util.packed.PackedInts.getReaderNoHeader(data, util.packed.PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue); PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue); ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed()); return(new NumericDocValuesAnonymousInnerClassHelper(this, decode, ordsReader)); case DELTA_COMPRESSED: //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int blockSize = data.readVInt(); int blockSize = data.readVInt(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final util.packed.BlockPackedReader reader = new util.packed.BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false); BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false); ramBytesUsed_Renamed.addAndGet(reader.ramBytesUsed()); return(reader); case UNCOMPRESSED: //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final byte bytes[] = new byte[maxDoc]; sbyte[] bytes = new sbyte[maxDoc]; data.readBytes(bytes, 0, bytes.Length); ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(bytes)); return(new NumericDocValuesAnonymousInnerClassHelper2(this, bytes)); case GCD_COMPRESSED: //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final long min = data.readLong(); long min = data.readLong(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final long mult = data.readLong(); long mult = data.readLong(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int quotientBlockSize = data.readVInt(); int quotientBlockSize = data.readVInt(); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final util.packed.BlockPackedReader quotientReader = new util.packed.BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false); BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false); ramBytesUsed_Renamed.addAndGet(quotientReader.ramBytesUsed()); return(new NumericDocValuesAnonymousInnerClassHelper3(this, min, mult, quotientReader)); default: throw new AssertionError(); } }
public override long GetSizeInBytes() { long mem = RamUsageEstimator.ShallowSizeOf(this) + RamUsageEstimator.ShallowSizeOf(normalCompletion) + RamUsageEstimator.ShallowSizeOf(higherWeightsCompletion); if (normalCompletion != null) { mem += normalCompletion.FST.GetSizeInBytes(); } if (higherWeightsCompletion != null && (normalCompletion == null || normalCompletion.FST != higherWeightsCompletion.FST)) { // the fst should be shared between the 2 completion instances, don't count it twice mem += higherWeightsCompletion.FST.GetSizeInBytes(); } return(mem); }
public override long RamBytesUsed() { return(RamUsageEstimator.AlignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 2 * RamUsageEstimator.NUM_BYTES_INT + RamUsageEstimator.NUM_BYTES_OBJECT_REF) + RamUsageEstimator.SizeOf(Blocks)); // blocks ref - valueCount,bitsPerValue }
/// <summary> /// Expert: instantiates a new reader. </summary> protected internal Lucene45DocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension) { string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension); // read in the entries from the metadata file. ChecksumIndexInput @in = state.Directory.OpenChecksumInput(metaName, state.Context); this.maxDoc = state.SegmentInfo.DocCount; bool success = false; try { version = CodecUtil.CheckHeader(@in, metaCodec, Lucene45DocValuesFormat.VERSION_START, Lucene45DocValuesFormat.VERSION_CURRENT); numerics = new Dictionary <int, NumericEntry>(); ords = new Dictionary <int, NumericEntry>(); ordIndexes = new Dictionary <int, NumericEntry>(); binaries = new Dictionary <int, BinaryEntry>(); sortedSets = new Dictionary <int, SortedSetEntry>(); ReadFields(@in /*, state.FieldInfos // LUCENENET: Not read */); if (version >= Lucene45DocValuesFormat.VERSION_CHECKSUM) { CodecUtil.CheckFooter(@in); } else { #pragma warning disable 612, 618 CodecUtil.CheckEOF(@in); #pragma warning restore 612, 618 } success = true; } finally { if (success) { IOUtils.Dispose(@in); } else { IOUtils.DisposeWhileHandlingException(@in); } } success = false; try { string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension); data = state.Directory.OpenInput(dataName, state.Context); int version2 = CodecUtil.CheckHeader(data, dataCodec, Lucene45DocValuesFormat.VERSION_START, Lucene45DocValuesFormat.VERSION_CURRENT); if (version != version2) { throw new Exception("Format versions mismatch"); } success = true; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(this.data); } } ramBytesUsed = new AtomicInt64(RamUsageEstimator.ShallowSizeOfInstance(this.GetType())); }
/// <summary> /// Returns total byte size used by cached filters. </summary> public virtual long GetSizeInBytes() { IList <DocIdSet> docIdSets = new List <DocIdSet>(_cache.Values); return(docIdSets.Sum(dis => RamUsageEstimator.SizeOf(dis))); }