//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void test() throws Exception
public virtual void test()
{
for (int i = 0; i < tests.Length; i += 3)
{
File f = new File(DICTIONARY_HOME, tests[i]);
Debug.Assert(f.exists());
using (ZipFile zip = new ZipFile(f, StandardCharsets.UTF_8))
{
ZipEntry dicEntry = zip.getEntry(tests[i + 1]);
Debug.Assert(dicEntry != null);
ZipEntry affEntry = zip.getEntry(tests[i + 2]);
Debug.Assert(affEntry != null);
using (System.IO.Stream dictionary = zip.getInputStream(dicEntry), System.IO.Stream affix = zip.getInputStream(affEntry))
{
Dictionary dic = new Dictionary(affix, dictionary);
Console.WriteLine(tests[i] + "\t" + RamUsageEstimator.humanSizeOf(dic) + "\t(" + "words=" + RamUsageEstimator.humanSizeOf(dic.words) + ", " + "flags=" + RamUsageEstimator.humanSizeOf(dic.flagLookup) + ", " + "strips=" + RamUsageEstimator.humanSizeOf(dic.stripData) + ", " + "conditions=" + RamUsageEstimator.humanSizeOf(dic.patterns) + ", " + "affixData=" + RamUsageEstimator.humanSizeOf(dic.affixData) + ", " + "prefixes=" + RamUsageEstimator.humanSizeOf(dic.prefixes) + ", " + "suffixes=" + RamUsageEstimator.humanSizeOf(dic.suffixes) + ")");
}
}
}
}
/// <summary>
/// Returns a String representation of the index data for debugging purposes.
/// </summary>
/// <returns> the string representation </returns>
public override string ToString()
{
StringBuilder result = new StringBuilder(256);
SortFields();
int sumPositions = 0;
int sumTerms = 0;
BytesRef spare = new BytesRef();
for (int i = 0; i < sortedFields.Length; i++)
{
KeyValuePair <string, Info> entry = sortedFields[i];
string fieldName = entry.Key;
Info info = entry.Value;
info.SortTerms();
result.Append(fieldName + ":\n");
SliceByteStartArray sliceArray = info.sliceArray;
int numPositions = 0;
Int32BlockPool.SliceReader postingsReader = new Int32BlockPool.SliceReader(intBlockPool);
for (int j = 0; j < info.terms.Count; j++)
{
int ord = info.sortedTerms[j];
info.terms.Get(ord, spare);
int freq = sliceArray.freq[ord];
result.Append("\t'" + spare + "':" + freq + ":");
postingsReader.Reset(sliceArray.start[ord], sliceArray.end[ord]);
result.Append(" [");
int iters = storeOffsets ? 3 : 1;
while (!postingsReader.IsEndOfSlice)
{
result.Append("(");
for (int k = 0; k < iters; k++)
{
result.Append(postingsReader.ReadInt32());
if (k < iters - 1)
{
result.Append(", ");
}
}
result.Append(")");
if (!postingsReader.IsEndOfSlice)
{
result.Append(",");
}
}
result.Append("]");
result.Append("\n");
numPositions += freq;
}
result.Append("\tterms=" + info.terms.Count);
result.Append(", positions=" + numPositions);
result.Append(", memory=" + RamUsageEstimator.HumanReadableUnits(RamUsageEstimator.SizeOf(info)));
result.Append("\n");
sumPositions += numPositions;
sumTerms += info.terms.Count;
}
result.Append("\nfields=" + sortedFields.Length);
result.Append(", terms=" + sumTerms);
result.Append(", positions=" + sumPositions);
result.Append(", memory=" + RamUsageEstimator.HumanReadableUnits(GetMemorySize()));
return(result.ToString());
}
/// <summary>
/// Return the number of bytes used by this instance. </summary>
public virtual long RamBytesUsed()
{
// TODO: this is called per-doc-per-norms/dv-field, can we optimize this?
long bytesUsed = RamUsageEstimator.AlignObjectSize(BaseRamBytesUsed()) + (pending != null ? RamUsageEstimator.SizeOf(pending) : 0L) + RamUsageEstimator.AlignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)RamUsageEstimator.NUM_BYTES_OBJECT_REF * values.Length); // values
return(bytesUsed + valuesBytes);
}
private long DocsWithFieldBytesUsed()
{
// size of the long[] + some overhead
return(RamUsageEstimator.SizeOf(docsWithField.Bits) + 64);
}
/// <summary>
/// Returns a reasonable approximation of the main memory [bytes] consumed by
/// this instance. Useful for smart memory sensititive caches/pools. </summary>
/// <returns> the main memory consumption </returns>
public virtual long GetMemorySize()
{
return(RamUsageEstimator.SizeOf(this));
}
/// <summary> Computes (and stores) the estimated size of the cache Value </summary>
/// <seealso cref="EstimatedSize">
/// </seealso>
public virtual void EstimateSize(RamUsageEstimator ramCalc)
{
long size = ramCalc.EstimateRamUsage(Value);
EstimatedSize = RamUsageEstimator.HumanReadableUnits(size, new System.Globalization.NumberFormatInfo()); // {{Aroush-2.9}} in Java, the formater is set to "0.#", so we need to do the same in C#
}
public override long RamBytesUsed()
{
return(base.RamBytesUsed() + RamUsageEstimator.SizeOf(averages) + RamUsageEstimator.SizeOf(minValues));
}
/// <summary>
/// Computes (and stores) the estimated size of the cache <see cref="Value"/>
/// </summary>
/// <seealso cref="EstimatedSize"/>
public void EstimateSize()
{
long bytesUsed = RamUsageEstimator.SizeOf(Value);
size = RamUsageEstimator.HumanReadableUnits(bytesUsed);
}
public override long RamBytesUsed()
{
return(RamUsageEstimator.AlignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 3 * RamUsageEstimator.NUM_BYTES_INT + RamUsageEstimator.NUM_BYTES_LONG + RamUsageEstimator.NUM_BYTES_OBJECT_REF) + RamUsageEstimator.SizeOf(Blocks)); // blocks ref - maskRight - bpvMinusBlockSize,valueCount,bitsPerValue
}
public virtual long RamBytesUsed()
{
return(RamUsageEstimator.SizeOf(_filter.GetBits()));
}
/// <summary>
/// Loads the segment information at segment load time.
/// </summary>
/// <param name="indexEnum">
/// the term enum. </param>
/// <param name="indexDivisor">
/// the index divisor. </param>
/// <param name="tiiFileLength">
/// the size of the tii file, used to approximate the size of the
/// buffer. </param>
/// <param name="totalIndexInterval">
/// the total index interval. </param>
public TermInfosReaderIndex(SegmentTermEnum indexEnum, int indexDivisor, long tiiFileLength, int totalIndexInterval)
{
this.TotalIndexInterval = totalIndexInterval;
IndexSize = 1 + ((int)indexEnum.Size - 1) / indexDivisor;
SkipInterval = indexEnum.SkipInterval;
// this is only an inital size, it will be GCed once the build is complete
long initialSize = (long)(tiiFileLength * 1.5) / indexDivisor;
PagedBytes dataPagedBytes = new PagedBytes(EstimatePageBits(initialSize));
PagedBytesDataOutput dataOutput = dataPagedBytes.DataOutput;
int bitEstimate = 1 + MathUtil.Log(tiiFileLength, 2);
GrowableWriter indexToTerms = new GrowableWriter(bitEstimate, IndexSize, PackedInts.DEFAULT);
string currentField = null;
IList <string> fieldStrs = new List <string>();
int fieldCounter = -1;
for (int i = 0; indexEnum.Next(); i++)
{
Term term = indexEnum.Term();
if (currentField == null || !currentField.Equals(term.Field))
{
currentField = term.Field;
fieldStrs.Add(currentField);
fieldCounter++;
}
TermInfo termInfo = indexEnum.TermInfo();
indexToTerms.Set(i, dataOutput.Position);
dataOutput.WriteVInt(fieldCounter);
dataOutput.WriteString(term.Text());
dataOutput.WriteVInt(termInfo.DocFreq);
if (termInfo.DocFreq >= SkipInterval)
{
dataOutput.WriteVInt(termInfo.SkipOffset);
}
dataOutput.WriteVLong(termInfo.FreqPointer);
dataOutput.WriteVLong(termInfo.ProxPointer);
dataOutput.WriteVLong(indexEnum.IndexPointer);
for (int j = 1; j < indexDivisor; j++)
{
if (!indexEnum.Next())
{
break;
}
}
}
Fields = new Term[fieldStrs.Count];
for (int i = 0; i < Fields.Length; i++)
{
Fields[i] = new Term(fieldStrs[i]);
}
dataPagedBytes.Freeze(true);
DataInput = dataPagedBytes.DataInput;
IndexToDataOffset = indexToTerms.Mutable;
RamBytesUsed_Renamed = Fields.Length * (RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.ShallowSizeOfInstance(typeof(Term))) + dataPagedBytes.RamBytesUsed() + IndexToDataOffset.RamBytesUsed();
}
internal Lucene42DocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension)
{
MaxDoc = state.SegmentInfo.DocCount;
string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension);
// read in the entries from the metadata file.
ChecksumIndexInput @in = state.Directory.OpenChecksumInput(metaName, state.Context);
bool success = false;
RamBytesUsed_Renamed = new AtomicLong(RamUsageEstimator.ShallowSizeOfInstance(this.GetType()));
try
{
Version = CodecUtil.CheckHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT);
Numerics = new Dictionary <int, NumericEntry>();
Binaries = new Dictionary <int, BinaryEntry>();
Fsts = new Dictionary <int, FSTEntry>();
ReadFields(@in, state.FieldInfos);
if (Version >= VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(@in);
}
else
{
CodecUtil.CheckEOF(@in);
}
success = true;
}
finally
{
if (success)
{
IOUtils.Close(@in);
}
else
{
IOUtils.CloseWhileHandlingException(@in);
}
}
success = false;
try
{
string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension);
Data = state.Directory.OpenInput(dataName, state.Context);
int version2 = CodecUtil.CheckHeader(Data, dataCodec, VERSION_START, VERSION_CURRENT);
if (Version != version2)
{
throw new CorruptIndexException("Format versions mismatch");
}
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(this.Data);
}
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: MemoryDocValuesProducer(index.SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws java.io.IOException
internal MemoryDocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension)
{
maxDoc = state.segmentInfo.DocCount;
string metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
// read in the entries from the metadata file.
ChecksumIndexInput @in = state.directory.openChecksumInput(metaName, state.context);
bool success = false;
try
{
version = CodecUtil.checkHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT);
numerics = new Dictionary <>();
binaries = new Dictionary <>();
fsts = new Dictionary <>();
readFields(@in, state.fieldInfos);
if (version >= VERSION_CHECKSUM)
{
CodecUtil.checkFooter(@in);
}
else
{
CodecUtil.checkEOF(@in);
}
ramBytesUsed_Renamed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(this.GetType()));
success = true;
}
finally
{
if (success)
{
IOUtils.close(@in);
}
else
{
IOUtils.closeWhileHandlingException(@in);
}
}
success = false;
try
{
string dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.openInput(dataName, state.context);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int version2 = codecs.CodecUtil.checkHeader(data, dataCodec, VERSION_START, VERSION_CURRENT);
int version2 = CodecUtil.checkHeader(data, dataCodec, VERSION_START, VERSION_CURRENT);
if (version != version2)
{
throw new CorruptIndexException("Format versions mismatch");
}
success = true;
}
finally
{
if (!success)
{
IOUtils.closeWhileHandlingException(this.data);
}
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private index.NumericDocValues loadNumeric(index.FieldInfo field) throws java.io.IOException
private NumericDocValues loadNumeric(FieldInfo field)
{
NumericEntry entry = numerics[field.number];
data.seek(entry.offset + entry.missingBytes);
switch (entry.format)
{
case TABLE_COMPRESSED:
int size = data.readVInt();
if (size > 256)
{
throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values, input=" + data);
}
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final long decode[] = new long[size];
long[] decode = new long[size];
for (int i = 0; i < decode.Length; i++)
{
decode[i] = data.readLong();
}
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int formatID = data.readVInt();
int formatID = data.readVInt();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int bitsPerValue = data.readVInt();
int bitsPerValue = data.readVInt();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final util.packed.PackedInts.Reader ordsReader = util.packed.PackedInts.getReaderNoHeader(data, util.packed.PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue);
PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue);
ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
return(new NumericDocValuesAnonymousInnerClassHelper(this, decode, ordsReader));
case DELTA_COMPRESSED:
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int blockSize = data.readVInt();
int blockSize = data.readVInt();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final util.packed.BlockPackedReader reader = new util.packed.BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false);
BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false);
ramBytesUsed_Renamed.addAndGet(reader.ramBytesUsed());
return(reader);
case UNCOMPRESSED:
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final byte bytes[] = new byte[maxDoc];
sbyte[] bytes = new sbyte[maxDoc];
data.readBytes(bytes, 0, bytes.Length);
ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(bytes));
return(new NumericDocValuesAnonymousInnerClassHelper2(this, bytes));
case GCD_COMPRESSED:
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final long min = data.readLong();
long min = data.readLong();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final long mult = data.readLong();
long mult = data.readLong();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int quotientBlockSize = data.readVInt();
int quotientBlockSize = data.readVInt();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final util.packed.BlockPackedReader quotientReader = new util.packed.BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false);
BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false);
ramBytesUsed_Renamed.addAndGet(quotientReader.ramBytesUsed());
return(new NumericDocValuesAnonymousInnerClassHelper3(this, min, mult, quotientReader));
default:
throw new AssertionError();
}
}
public override long GetSizeInBytes()
{
long mem = RamUsageEstimator.ShallowSizeOf(this) + RamUsageEstimator.ShallowSizeOf(normalCompletion) + RamUsageEstimator.ShallowSizeOf(higherWeightsCompletion);
if (normalCompletion != null)
{
mem += normalCompletion.FST.GetSizeInBytes();
}
if (higherWeightsCompletion != null && (normalCompletion == null || normalCompletion.FST != higherWeightsCompletion.FST))
{
// the fst should be shared between the 2 completion instances, don't count it twice
mem += higherWeightsCompletion.FST.GetSizeInBytes();
}
return(mem);
}
public override long RamBytesUsed()
{
return(RamUsageEstimator.AlignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 2 * RamUsageEstimator.NUM_BYTES_INT + RamUsageEstimator.NUM_BYTES_OBJECT_REF) + RamUsageEstimator.SizeOf(Blocks)); // blocks ref - valueCount,bitsPerValue
}
/// <summary>
/// Expert: instantiates a new reader. </summary>
protected internal Lucene45DocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension)
{
string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension);
// read in the entries from the metadata file.
ChecksumIndexInput @in = state.Directory.OpenChecksumInput(metaName, state.Context);
this.maxDoc = state.SegmentInfo.DocCount;
bool success = false;
try
{
version = CodecUtil.CheckHeader(@in, metaCodec, Lucene45DocValuesFormat.VERSION_START, Lucene45DocValuesFormat.VERSION_CURRENT);
numerics = new Dictionary <int, NumericEntry>();
ords = new Dictionary <int, NumericEntry>();
ordIndexes = new Dictionary <int, NumericEntry>();
binaries = new Dictionary <int, BinaryEntry>();
sortedSets = new Dictionary <int, SortedSetEntry>();
ReadFields(@in /*, state.FieldInfos // LUCENENET: Not read */);
if (version >= Lucene45DocValuesFormat.VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(@in);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(@in);
#pragma warning restore 612, 618
}
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(@in);
}
else
{
IOUtils.DisposeWhileHandlingException(@in);
}
}
success = false;
try
{
string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension);
data = state.Directory.OpenInput(dataName, state.Context);
int version2 = CodecUtil.CheckHeader(data, dataCodec, Lucene45DocValuesFormat.VERSION_START, Lucene45DocValuesFormat.VERSION_CURRENT);
if (version != version2)
{
throw new Exception("Format versions mismatch");
}
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(this.data);
}
}
ramBytesUsed = new AtomicInt64(RamUsageEstimator.ShallowSizeOfInstance(this.GetType()));
}
/// <summary>
/// Returns total byte size used by cached filters. </summary>
public virtual long GetSizeInBytes()
{
IList <DocIdSet> docIdSets = new List <DocIdSet>(_cache.Values);
return(docIdSets.Sum(dis => RamUsageEstimator.SizeOf(dis)));
}
