private void UpdateInCache(Uri uri, string withFile)
{
using (var progressHandler = EmulationManager.Instance.ProgressMonitor.Start("Updating cache"))
{
lock (CacheDirectory)
{
var index = ReadBinariesIndex();
BinaryEntry entry;
var fileId = 0;
if (!index.TryGetValue(uri, out entry))
{
foreach (var element in index)
{
fileId = Math.Max(fileId, element.Value.Index) + 1;
}
}
else
{
fileId = entry.Index;
}
FileCopier.Copy(withFile, GetBinaryFileName(fileId), true);
// checksum will be 'null' if the uri pattern does not contain
// checksum/size information
TryGetChecksumAndSizeFromUri(uri, out var checksum, out var size);
index[uri] = new BinaryEntry(fileId, size, checksum);
WriteBinariesIndex(index);
}
}
}
private void ReadFields(IndexInput meta /*, FieldInfos infos // LUCENENET: Not read */)
{
int fieldNumber = meta.ReadVInt32();
while (fieldNumber != -1)
{
// check should be: infos.fieldInfo(fieldNumber) != null, which incorporates negative check
// but docvalues updates are currently buggy here (loading extra stuff, etc): LUCENE-5616
if (fieldNumber < 0)
{
// trickier to validate more: because we re-use for norms, because we use multiple entries
// for "composite" types like sortedset, etc.
throw new Exception("Invalid field number: " + fieldNumber + " (resource=" + meta + ")");
}
byte type = meta.ReadByte();
if (type == Lucene45DocValuesFormat.NUMERIC)
{
numerics[fieldNumber] = ReadNumericEntry(meta);
}
else if (type == Lucene45DocValuesFormat.BINARY)
{
BinaryEntry b = ReadBinaryEntry(meta);
binaries[fieldNumber] = b;
}
else if (type == Lucene45DocValuesFormat.SORTED)
{
ReadSortedField(fieldNumber, meta /*, infos // LUCENENET: Never read */);
}
else if (type == Lucene45DocValuesFormat.SORTED_SET)
{
SortedSetEntry ss = ReadSortedSetEntry(meta);
sortedSets[fieldNumber] = ss;
if (ss.Format == Lucene45DocValuesConsumer.SORTED_SET_WITH_ADDRESSES)
{
ReadSortedSetFieldWithAddresses(fieldNumber, meta /*, infos // LUCENENET: Never read */);
}
else if (ss.Format == Lucene45DocValuesConsumer.SORTED_SET_SINGLE_VALUED_SORTED)
{
if (meta.ReadVInt32() != fieldNumber)
{
throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
if (meta.ReadByte() != Lucene45DocValuesFormat.SORTED)
{
throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
ReadSortedField(fieldNumber, meta /*, infos // LUCENENET: Never read */);
}
else
{
throw new Exception();
}
}
else
{
throw new Exception("invalid type: " + type + ", resource=" + meta);
}
fieldNumber = meta.ReadVInt32();
}
}
protected override MonotonicBlockPackedReader GetAddressInstance(IndexInput data, FieldInfo field,
BinaryEntry bytes)
{
data.Seek(bytes.AddressesOffset);
return new MonotonicBlockPackedReader((IndexInput)data.Clone(), bytes.PackedIntsVersion, bytes.BlockSize, bytes.Count,
true);
}
private BinaryDocValues LoadBinary(FieldInfo field)
{
BinaryEntry entry = binaries[field.Number];
data.Seek(entry.offset);
var bytes = new PagedBytes(16);
bytes.Copy(data, entry.numBytes);
var bytesReader = bytes.Freeze(true);
if (entry.minLength == entry.maxLength)
{
int fixedLength = entry.minLength;
ramBytesUsed.AddAndGet(bytes.RamBytesUsed());
return(new BinaryDocValuesAnonymousClass(bytesReader, fixedLength));
}
else
{
data.Seek(data.Position + entry.missingBytes); // LUCENENET specific: Renamed from getFilePointer() to match FileStream
var addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion,
entry.blockSize, maxDoc, false);
ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + addresses.RamBytesUsed());
return(new BinaryDocValuesAnonymousClass2(bytesReader, addresses));
}
}
private bool UpdateInCache(Uri uri, string withFile)
{
using (var progressHandler = EmulationManager.Instance.ProgressMonitor.Start("Updating cache"))
{
lock (CacheDirectory)
{
var index = ReadBinariesIndex();
BinaryEntry entry;
var fileId = 0;
if (!index.TryGetValue(uri, out entry))
{
foreach (var element in index)
{
fileId = Math.Max(fileId, element.Value.Index) + 1;
}
}
else
{
fileId = entry.Index;
}
FileCopier.Copy(withFile, GetBinaryFileName(fileId), true);
long size;
var checksum = GetChecksumAndSizeFromUri(uri, out size);
entry = new BinaryEntry(fileId, size, checksum);
if (!Verify(withFile, entry))
{
return(false);
}
index[uri] = entry;
WriteBinariesIndex(index);
return(true);
}
}
}
private void ReadSortedField(int fieldNumber, IndexInput meta /*, FieldInfos infos // LUCENENET: Never read */)
{
// sorted = binary + numeric
if (meta.ReadVInt32() != fieldNumber)
{
throw new Exception("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
if (meta.ReadByte() != Lucene45DocValuesFormat.BINARY)
{
throw new Exception("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
BinaryEntry b = ReadBinaryEntry(meta);
binaries[fieldNumber] = b;
if (meta.ReadVInt32() != fieldNumber)
{
throw new Exception("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
if (meta.ReadByte() != Lucene45DocValuesFormat.NUMERIC)
{
throw new Exception("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
NumericEntry n = ReadNumericEntry(meta);
ords[fieldNumber] = n;
}
internal static BinaryEntry ReadBinaryEntry(IndexInput meta)
{
BinaryEntry entry = new BinaryEntry();
entry.format = meta.ReadVInt32();
entry.missingOffset = meta.ReadInt64();
entry.minLength = meta.ReadVInt32();
entry.maxLength = meta.ReadVInt32();
entry.Count = meta.ReadVInt64();
entry.offset = meta.ReadInt64();
switch (entry.format)
{
case Lucene45DocValuesConsumer.BINARY_FIXED_UNCOMPRESSED:
break;
case Lucene45DocValuesConsumer.BINARY_PREFIX_COMPRESSED:
entry.AddressInterval = meta.ReadVInt32();
entry.AddressesOffset = meta.ReadInt64();
entry.PackedInt32sVersion = meta.ReadVInt32();
entry.BlockSize = meta.ReadVInt32();
break;
case Lucene45DocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED:
entry.AddressesOffset = meta.ReadInt64();
entry.PackedInt32sVersion = meta.ReadVInt32();
entry.BlockSize = meta.ReadVInt32();
break;
default:
throw new Exception("Unknown format: " + entry.format + ", input=" + meta);
}
return(entry);
}
protected override MonotonicBlockPackedReader GetAddressInstance(IndexInput data, FieldInfo field,
BinaryEntry bytes)
{
data.Seek(bytes.AddressesOffset);
return(new MonotonicBlockPackedReader((IndexInput)data.Clone(), bytes.PackedIntsVersion, bytes.BlockSize, bytes.Count,
true));
}
private BinaryDocValues LoadBinary(FieldInfo field)
{
BinaryEntry entry = binaries[field.Number];
data.Seek(entry.offset);
var bytes = new PagedBytes(16);
bytes.Copy(data, entry.numBytes);
var bytesReader = bytes.Freeze(true);
if (entry.minLength == entry.maxLength)
{
int fixedLength = entry.minLength;
ramBytesUsed.AddAndGet(bytes.RamBytesUsed());
return(new BinaryDocValuesAnonymousInnerClassHelper(this, bytesReader, fixedLength));
}
else
{
data.Seek(data.GetFilePointer() + entry.missingBytes);
var addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion,
entry.blockSize, maxDoc, false);
ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + addresses.RamBytesUsed());
return(new BinaryDocValuesAnonymousInnerClassHelper2(this, bytesReader, addresses));
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private index.BinaryDocValues loadBinary(index.FieldInfo field) throws java.io.IOException
private BinaryDocValues loadBinary(FieldInfo field)
{
BinaryEntry entry = binaries[field.number];
data.seek(entry.offset);
PagedBytes bytes = new PagedBytes(16);
bytes.copy(data, entry.numBytes);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final util.PagedBytes.Reader bytesReader = bytes.freeze(true);
PagedBytes.Reader bytesReader = bytes.freeze(true);
if (entry.minLength == entry.maxLength)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int fixedLength = entry.minLength;
int fixedLength = entry.minLength;
ramBytesUsed_Renamed.addAndGet(bytes.ramBytesUsed());
return(new BinaryDocValuesAnonymousInnerClassHelper(this, bytesReader, fixedLength));
}
else
{
data.seek(data.FilePointer + entry.missingBytes);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final util.packed.MonotonicBlockPackedReader addresses = new util.packed.MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
ramBytesUsed_Renamed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed());
return(new BinaryDocValuesAnonymousInnerClassHelper2(this, bytesReader, addresses));
}
}
private bool Verify(string fileName, BinaryEntry entry)
{
if (!File.Exists(fileName))
{
Logger.LogAs(this, LogLevel.Warning, "Binary {0} found in index but is missing in cache.", fileName);
return(false);
}
if (entry.Checksum != null)
{
var actualSize = new FileInfo(fileName).Length;
if (actualSize != entry.Size)
{
Logger.LogAs(this, LogLevel.Warning, "Size of the file differs: is {0}B, should be {1}B.", actualSize, entry.Size);
return(false);
}
if (ConfigurationManager.Instance.Get("file-fetcher", "calculate-checksum", true))
{
byte[] checksum;
using (var progressHandler = EmulationManager.Instance.ProgressMonitor.Start("Calculating SHA1 checksum..."))
{
checksum = GetSHA1Checksum(fileName);
}
if (!checksum.SequenceEqual(entry.Checksum))
{
Logger.LogAs(this, LogLevel.Warning, "Checksum of the file differs, is {0}, should be {1}.", ChecksumToText(checksum), ChecksumToText(entry.Checksum));
return(false);
}
}
}
return(true);
}
private BinaryDocValues GetCompressedBinary(FieldInfo field, BinaryEntry bytes)
{
IndexInput data = (IndexInput)this.data.Clone();
MonotonicBlockPackedReader addresses = GetIntervalInstance(data, field, bytes);
return(new CompressedBinaryDocValues(bytes, addresses, data));
}
private BinaryDocValues GetVariableBinary(FieldInfo field, BinaryEntry bytes)
{
IndexInput data = (IndexInput)this.data.Clone();
MonotonicBlockPackedReader addresses = GetAddressInstance(data, field, bytes);
return(new Int64BinaryDocValuesAnonymousInnerClassHelper2(bytes, data, addresses));
}
public CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, IndexInput data)
{
this.bytes = bytes;
this.interval = bytes.AddressInterval;
this.addresses = addresses;
this.data = data;
this.numValues = bytes.Count;
this.numIndexValues = addresses.Count;
this.termsEnum = GetTermsEnum(data);
}
public CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, IndexInput data)
{
this.Bytes = bytes;
this.Interval = bytes.AddressInterval;
this.Addresses = addresses;
this.Data = data;
this.NumValues = bytes.Count;
this.NumIndexValues = addresses.Size();
this.TermsEnum_Renamed = GetTermsEnum(data);
}
private bool VerifyCachedFile(string fileName, BinaryEntry entry)
{
if (!File.Exists(fileName))
{
Logger.LogAs(this, LogLevel.Warning, "Binary {0} found in index but is missing in cache.", fileName);
return(false);
}
if (entry.Checksum == null)
{
return(true);
}
return(VerifySize(fileName, entry.Size) && VerifyChecksum(fileName, entry.Checksum));
}
private static BinaryEntry ReadBinaryEntry(IndexInput meta) // LUCENENET: CA1822: Mark members as static
{
var entry = new BinaryEntry();
entry.offset = meta.ReadInt64();
entry.numBytes = meta.ReadInt32();
entry.count = meta.ReadInt32();
entry.missingOffset = meta.ReadInt64();
if (entry.missingOffset != -1)
{
entry.missingBytes = meta.ReadInt64();
}
else
{
entry.missingBytes = 0;
}
return(entry);
}
public override BinaryDocValues GetBinary(FieldInfo field)
{
BinaryEntry bytes = binaries[field.Number];
switch (bytes.format)
{
case Lucene45DocValuesConsumer.BINARY_FIXED_UNCOMPRESSED:
return(GetFixedBinary(/*field, LUCENENET: Never read */ bytes));
case Lucene45DocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED:
return(GetVariableBinary(field, bytes));
case Lucene45DocValuesConsumer.BINARY_PREFIX_COMPRESSED:
return(GetCompressedBinary(field, bytes));
default:
throw new Exception();
}
}
private BinaryEntry ReadBinaryEntry(IndexInput meta)
{
var entry = new BinaryEntry();
entry.offset = meta.ReadLong();
entry.numBytes = meta.ReadInt();
entry.count = meta.ReadInt();
entry.missingOffset = meta.ReadLong();
if (entry.missingOffset != -1)
{
entry.missingBytes = meta.ReadLong();
}
else
{
entry.missingBytes = 0;
}
return(entry);
}
private BinaryDocValues LoadBinary(BinaryEntry entry)
{
data.Seek(entry.offset);
var bytes = new byte[entry.numBytes];
data.ReadBytes(bytes, 0, entry.numBytes);
data.Seek(entry.offset + entry.numBytes + entry.missingBytes);
var address = new int[entry.count + 1];
for (int i = 0; i < entry.count; i++)
{
address[i] = data.ReadInt();
}
address[entry.count] = data.ReadInt();
ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(bytes) + RamUsageEstimator.SizeOf(address));
return(new BinaryDocValuesAnonymousInnerClassHelper(bytes, address));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private index.BinaryDocValues loadBinary(BinaryEntry entry) throws java.io.IOException
private BinaryDocValues loadBinary(BinaryEntry entry)
{
data.seek(entry.offset);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final byte[] bytes = new byte[entry.numBytes];
sbyte[] bytes = new sbyte[entry.numBytes];
data.readBytes(bytes, 0, entry.numBytes);
data.seek(entry.offset + entry.numBytes + entry.missingBytes);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int[] address = new int[entry.count+1];
int[] address = new int[entry.count + 1];
for (int i = 0; i < entry.count; i++)
{
address[i] = data.readInt();
}
address[entry.count] = data.readInt();
ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(bytes) + RamUsageEstimator.sizeOf(address));
return(new BinaryDocValuesAnonymousInnerClassHelper(this, bytes, address));
}
private BinaryDocValues LoadBinary(FieldInfo field)
{
BinaryEntry entry = binaries[field.Number];
data.Seek(entry.Offset);
PagedBytes bytes = new PagedBytes(16);
bytes.Copy(data, entry.NumBytes);
PagedBytes.Reader bytesReader = bytes.Freeze(true);
if (entry.MinLength == entry.MaxLength)
{
int fixedLength = entry.MinLength;
ramBytesUsed.AddAndGet(bytes.RamBytesUsed());
return(new BinaryDocValuesAnonymousInnerClassHelper(bytesReader, fixedLength));
}
else
{
MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.PackedInt32sVersion, entry.BlockSize, maxDoc, false);
ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + addresses.RamBytesUsed());
return(new BinaryDocValuesAnonymousInnerClassHelper2(bytesReader, addresses));
}
}
private void ReadSortedSetFieldWithAddresses(int fieldNumber, IndexInput meta, FieldInfos infos)
{
// sortedset = binary + numeric (addresses) + ordIndex
if (meta.ReadVInt32() != fieldNumber)
{
throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
if (meta.ReadByte() != Lucene45DocValuesFormat.BINARY)
{
throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
BinaryEntry b = ReadBinaryEntry(meta);
binaries[fieldNumber] = b;
if (meta.ReadVInt32() != fieldNumber)
{
throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
if (meta.ReadByte() != Lucene45DocValuesFormat.NUMERIC)
{
throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
NumericEntry n1 = ReadNumericEntry(meta);
ords[fieldNumber] = n1;
if (meta.ReadVInt32() != fieldNumber)
{
throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
if (meta.ReadByte() != Lucene45DocValuesFormat.NUMERIC)
{
throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
}
NumericEntry n2 = ReadNumericEntry(meta);
ordIndexes[fieldNumber] = n2;
}
public override Bits GetDocsWithField(FieldInfo field)
{
switch (field.DocValuesType)
{
case FieldInfo.DocValuesType_e.SORTED_SET:
return(DocValues.DocsWithValue(GetSortedSet(field), maxDoc));
case FieldInfo.DocValuesType_e.SORTED:
return(DocValues.DocsWithValue(GetSorted(field), maxDoc));
case FieldInfo.DocValuesType_e.BINARY:
BinaryEntry be = binaries[field.Number];
return(GetMissingBits(field.Number, be.missingOffset, be.missingBytes));
case FieldInfo.DocValuesType_e.NUMERIC:
NumericEntry ne = numerics[field.Number];
return(GetMissingBits(field.Number, ne.missingOffset, ne.missingBytes));
default:
throw new System.ArgumentOutOfRangeException();
}
}
public override IBits GetDocsWithField(FieldInfo field)
{
switch (field.DocValuesType)
{
case DocValuesType.SORTED_SET:
return(DocValues.DocsWithValue(GetSortedSet(field), maxDoc));
case DocValuesType.SORTED:
return(DocValues.DocsWithValue(GetSorted(field), maxDoc));
case DocValuesType.BINARY:
BinaryEntry be = binaries[field.Number];
return(GetMissingBits(be.missingOffset));
case DocValuesType.NUMERIC:
NumericEntry ne = numerics[field.Number];
return(GetMissingBits(ne.missingOffset));
default:
throw new InvalidOperationException();
}
}
public override IBits GetDocsWithField(FieldInfo field)
{
switch (field.DocValuesType)
{
case DocValuesType.SORTED_SET:
return(DocValues.DocsWithValue(GetSortedSet(field), maxDoc));
case DocValuesType.SORTED:
return(DocValues.DocsWithValue(GetSorted(field), maxDoc));
case DocValuesType.BINARY:
BinaryEntry be = binaries[field.Number];
return(GetMissingBits(field.Number, be.missingOffset, be.missingBytes));
case DocValuesType.NUMERIC:
NumericEntry ne = numerics[field.Number];
return(GetMissingBits(field.Number, ne.missingOffset, ne.missingBytes));
default:
throw AssertionError.Create();
}
}
private BinaryDocValues GetFixedBinary(/* FieldInfo field, // LUCENENET: Never read */ BinaryEntry bytes)
{
IndexInput data = (IndexInput)this.data.Clone();
return(new Int64BinaryDocValuesAnonymousInnerClassHelper(bytes, data));
}
private bool UpdateInCache(Uri uri, string withFile)
{
using(var progressHandler = EmulationManager.Instance.ProgressMonitor.Start("Updating cache"))
{
lock(CacheDirectory)
{
var index = ReadBinariesIndex();
BinaryEntry entry;
var fileId = 0;
if(!index.TryGetValue(uri, out entry))
{
foreach(var element in index)
{
fileId = Math.Max(fileId, element.Value.Index) + 1;
}
}
else
{
fileId = entry.Index;
}
FileCopier.Copy(withFile, GetBinaryFileName(fileId), true);
long size;
var checksum = GetChecksumAndSizeFromUri(uri, out size);
entry = new BinaryEntry(fileId, size, checksum);
if(!Verify(withFile, entry))
{
return false;
}
index[uri] = entry;
WriteBinariesIndex(index);
return true;
}
}
}
public static Entry CreateEntry( string name, object value, string defaultNamespace, string defaultDeclaringClass )
{
string stringValue = value as string;
System.Drawing.Bitmap bitmapValue = value as System.Drawing.Bitmap;
byte[] rawValue = value as byte[];
Entry entry = null;
if(stringValue != null)
{
entry = new StringEntry( name, stringValue );
}
else if(bitmapValue != null)
{
entry = new BitmapEntry( name, bitmapValue );
}
if(rawValue != null)
{
entry = TinyResourcesEntry.TryCreateTinyResourcesEntry( name, rawValue );
if(entry == null)
{
entry = new BinaryEntry( name, rawValue );
}
}
if(entry == null)
{
throw new Exception();
}
if(entry.Namespace.Length == 0)
{
entry.Namespace = defaultNamespace;
}
if(entry.ClassName.Length == 0)
{
ResourceTypeDescription typeDescription = ResourceTypeDescriptionFromResourceType( entry.ResourceType );
entry.ClassName = typeDescription.defaultEnum;
if(!string.IsNullOrEmpty( defaultDeclaringClass ))
{
entry.ClassName = string.Format( "{0}+{1}", defaultDeclaringClass, entry.ClassName );
}
}
return entry;
}
internal static BinaryEntry ReadBinaryEntry(IndexInput meta)
{
BinaryEntry entry = new BinaryEntry();
entry.Format = meta.ReadVInt();
entry.MissingOffset = meta.ReadLong();
entry.MinLength = meta.ReadVInt();
entry.MaxLength = meta.ReadVInt();
entry.Count = meta.ReadVLong();
entry.Offset = meta.ReadLong();
switch (entry.Format)
{
case Lucene45DocValuesConsumer.BINARY_FIXED_UNCOMPRESSED:
break;
case Lucene45DocValuesConsumer.BINARY_PREFIX_COMPRESSED:
entry.AddressInterval = meta.ReadVInt();
entry.AddressesOffset = meta.ReadLong();
entry.PackedIntsVersion = meta.ReadVInt();
entry.BlockSize = meta.ReadVInt();
break;
case Lucene45DocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED:
entry.AddressesOffset = meta.ReadLong();
entry.PackedIntsVersion = meta.ReadVInt();
entry.BlockSize = meta.ReadVInt();
break;
default:
throw new Exception("Unknown format: " + entry.Format + ", input=" + meta);
}
return entry;
}
/// <summary>
/// returns an address instance for prefix-compressed binary values.
/// @lucene.internal
/// </summary>
protected internal virtual MonotonicBlockPackedReader GetIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes)
{
MonotonicBlockPackedReader addresses;
long interval = bytes.AddressInterval;
lock (AddressInstances)
{
MonotonicBlockPackedReader addrInstance;
if (!AddressInstances.TryGetValue(field.Number, out addrInstance))
{
data.Seek(bytes.AddressesOffset);
long size;
if (bytes.Count % interval == 0)
{
size = bytes.Count / interval;
}
else
{
size = 1L + bytes.Count / interval;
}
addrInstance = new MonotonicBlockPackedReader(data, bytes.PackedIntsVersion, bytes.BlockSize, size, false);
AddressInstances[field.Number] = addrInstance;
RamBytesUsed_Renamed.AddAndGet(addrInstance.RamBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
}
addresses = addrInstance;
}
return addresses;
}
private BinaryDocValues LoadBinary(BinaryEntry entry)
{
data.Seek(entry.offset);
var bytes = new byte[entry.numBytes];
data.ReadBytes(bytes, 0, entry.numBytes);
data.Seek(entry.offset + entry.numBytes + entry.missingBytes);
var address = new int[entry.count + 1];
for (int i = 0; i < entry.count; i++)
{
address[i] = data.ReadInt();
}
address[entry.count] = data.ReadInt();
ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(bytes) + RamUsageEstimator.SizeOf(address));
return new BinaryDocValuesAnonymousInnerClassHelper(bytes, address);
}
private void ReadFields(IndexInput meta, FieldInfos infos)
{
int fieldNumber = meta.ReadVInt32();
while (fieldNumber != -1)
{
// check should be: infos.fieldInfo(fieldNumber) != null, which incorporates negative check
// but docvalues updates are currently buggy here (loading extra stuff, etc): LUCENE-5616
if (fieldNumber < 0)
{
// trickier to validate more: because we re-use for norms, because we use multiple entries
// for "composite" types like sortedset, etc.
throw new CorruptIndexException("Invalid field number: " + fieldNumber + ", input=" + meta);
}
int fieldType = meta.ReadByte();
if (fieldType == NUMBER)
{
var entry = new NumericEntry();
entry.Offset = meta.ReadInt64();
entry.Format = (sbyte)meta.ReadByte();
switch (entry.Format)
{
case DELTA_COMPRESSED:
case TABLE_COMPRESSED:
case GCD_COMPRESSED:
case UNCOMPRESSED:
break;
default:
throw new CorruptIndexException("Unknown format: " + entry.Format + ", input=" + meta);
}
if (entry.Format != UNCOMPRESSED)
{
entry.PackedInt32sVersion = meta.ReadVInt32();
}
numerics[fieldNumber] = entry;
}
else if (fieldType == BYTES)
{
BinaryEntry entry = new BinaryEntry();
entry.Offset = meta.ReadInt64();
entry.NumBytes = meta.ReadInt64();
entry.MinLength = meta.ReadVInt32();
entry.MaxLength = meta.ReadVInt32();
if (entry.MinLength != entry.MaxLength)
{
entry.PackedInt32sVersion = meta.ReadVInt32();
entry.BlockSize = meta.ReadVInt32();
}
binaries[fieldNumber] = entry;
}
else if (fieldType == FST)
{
FSTEntry entry = new FSTEntry();
entry.Offset = meta.ReadInt64();
entry.NumOrds = meta.ReadVInt64();
fsts[fieldNumber] = entry;
}
else
{
throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
}
fieldNumber = meta.ReadVInt32();
}
}
private void ReadFields(IndexInput meta, FieldInfos infos)
{
int fieldNumber = meta.ReadVInt();
while (fieldNumber != -1)
{
int fieldType = meta.ReadByte();
if (fieldType == NUMBER)
{
var entry = new NumericEntry {offset = meta.ReadLong(), missingOffset = meta.ReadLong()};
if (entry.missingOffset != -1)
{
entry.missingBytes = meta.ReadLong();
}
else
{
entry.missingBytes = 0;
}
entry.format = meta.ReadByte();
switch (entry.format)
{
case DELTA_COMPRESSED:
case TABLE_COMPRESSED:
case GCD_COMPRESSED:
case UNCOMPRESSED:
break;
default:
throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
}
if (entry.format != UNCOMPRESSED)
{
entry.packedIntsVersion = meta.ReadVInt();
}
numerics[fieldNumber] = entry;
}
else if (fieldType == BYTES)
{
var entry = new BinaryEntry
{
offset = meta.ReadLong(),
numBytes = meta.ReadLong(),
missingOffset = meta.ReadLong()
};
if (entry.missingOffset != -1)
{
entry.missingBytes = meta.ReadLong();
}
else
{
entry.missingBytes = 0;
}
entry.minLength = meta.ReadVInt();
entry.maxLength = meta.ReadVInt();
if (entry.minLength != entry.maxLength)
{
entry.packedIntsVersion = meta.ReadVInt();
entry.blockSize = meta.ReadVInt();
}
binaries[fieldNumber] = entry;
}
else if (fieldType == FST)
{
var entry = new FSTEntry {offset = meta.ReadLong(), numOrds = meta.ReadVLong()};
fsts[fieldNumber] = entry;
}
else
{
throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
}
fieldNumber = meta.ReadVInt();
}
}
private bool Verify(string fileName, BinaryEntry entry)
{
if(entry.Checksum != null)
{
long actualSize = -1;
try
{
actualSize = new FileInfo(fileName).Length;
}
catch (FileNotFoundException)
{
Logger.LogAs(this, LogLevel.Warning, "File {0} not found in cached binaries folder.", fileName);
return false;
}
if(actualSize != entry.Size)
{
Logger.LogAs(this, LogLevel.Warning, "Size of the file differs: is {0}B, should be {1}B.", actualSize, entry.Size);
return false;
}
if(ConfigurationManager.Instance.Get("file-fetcher", "calculate-checksum", true))
{
byte[] checksum;
using(var progressHandler = EmulationManager.Instance.ProgressMonitor.Start("Calculating SHA1 checksum..."))
{
checksum = GetSHA1Checksum(fileName);
}
if(!checksum.SequenceEqual(entry.Checksum))
{
Logger.LogAs(this, LogLevel.Warning, "Checksum of the file differs, is {0}, should be {1}.", ChecksumToText(checksum), ChecksumToText(entry.Checksum));
return false;
}
}
}
return true;
}
public CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, IndexInput data)
{
this.Bytes = bytes;
this.Interval = bytes.AddressInterval;
this.Addresses = addresses;
this.Data = data;
this.NumValues = bytes.Count;
this.NumIndexValues = addresses.Size();
this.TermsEnum_Renamed = GetTermsEnum(data);
}
private void ReadFields(IndexInput meta, FieldInfos infos)
{
int fieldNumber = meta.ReadVInt();
while (fieldNumber != -1)
{
// check should be: infos.fieldInfo(fieldNumber) != null, which incorporates negative check
// but docvalues updates are currently buggy here (loading extra stuff, etc): LUCENE-5616
if (fieldNumber < 0)
{
// trickier to validate more: because we re-use for norms, because we use multiple entries
// for "composite" types like sortedset, etc.
throw new CorruptIndexException("Invalid field number: " + fieldNumber + ", input=" + meta);
}
int fieldType = meta.ReadByte();
if (fieldType == NUMBER)
{
var entry = new NumericEntry {Offset = meta.ReadLong(), Format = (sbyte)meta.ReadByte()};
switch (entry.Format)
{
case DELTA_COMPRESSED:
case TABLE_COMPRESSED:
case GCD_COMPRESSED:
case UNCOMPRESSED:
break;
default:
throw new CorruptIndexException("Unknown format: " + entry.Format + ", input=" + meta);
}
if (entry.Format != UNCOMPRESSED)
{
entry.PackedIntsVersion = meta.ReadVInt();
}
Numerics[fieldNumber] = entry;
}
else if (fieldType == BYTES)
{
BinaryEntry entry = new BinaryEntry();
entry.Offset = meta.ReadLong();
entry.NumBytes = meta.ReadLong();
entry.MinLength = meta.ReadVInt();
entry.MaxLength = meta.ReadVInt();
if (entry.MinLength != entry.MaxLength)
{
entry.PackedIntsVersion = meta.ReadVInt();
entry.BlockSize = meta.ReadVInt();
}
Binaries[fieldNumber] = entry;
}
else if (fieldType == FST)
{
FSTEntry entry = new FSTEntry();
entry.Offset = meta.ReadLong();
entry.NumOrds = meta.ReadVLong();
Fsts[fieldNumber] = entry;
}
else
{
throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
}
fieldNumber = meta.ReadVInt();
}
}
/// <summary>
/// Returns an address instance for variable-length binary values.
/// <para/>
/// @lucene.internal
/// </summary>
protected virtual MonotonicBlockPackedReader GetAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes)
{
MonotonicBlockPackedReader addresses;
lock (addressInstances)
{
MonotonicBlockPackedReader addrInstance;
if (!addressInstances.TryGetValue(field.Number, out addrInstance) || addrInstance == null)
{
data.Seek(bytes.AddressesOffset);
addrInstance = new MonotonicBlockPackedReader(data, bytes.PackedInt32sVersion, bytes.BlockSize, bytes.Count, false);
addressInstances[field.Number] = addrInstance;
ramBytesUsed.AddAndGet(addrInstance.RamBytesUsed() + RamUsageEstimator.NUM_BYTES_INT32);
}
addresses = addrInstance;
}
return(addresses);
}
private BinaryDocValues GetFixedBinary(FieldInfo field, BinaryEntry bytes)
{
IndexInput data = (IndexInput)this.data.Clone();
return(new Int64BinaryDocValuesAnonymousInnerClassHelper(this, bytes, data));
}
/// <summary>
/// Returns an address instance for prefix-compressed binary values.
/// <para/>
/// @lucene.internal
/// </summary>
protected virtual MonotonicBlockPackedReader GetIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes)
{
MonotonicBlockPackedReader addresses;
long interval = bytes.AddressInterval;
lock (addressInstances)
{
if (!addressInstances.TryGetValue(field.Number, out MonotonicBlockPackedReader addrInstance))
{
data.Seek(bytes.AddressesOffset);
long size;
if (bytes.Count % interval == 0)
{
size = bytes.Count / interval;
}
else
{
size = 1L + bytes.Count / interval;
}
addrInstance = new MonotonicBlockPackedReader(data, bytes.PackedInt32sVersion, bytes.BlockSize, size, false);
addressInstances[field.Number] = addrInstance;
ramBytesUsed.AddAndGet(addrInstance.RamBytesUsed() + RamUsageEstimator.NUM_BYTES_INT32);
}
addresses = addrInstance;
}
return(addresses);
}
private BinaryDocValues GetVariableBinary(FieldInfo field, BinaryEntry bytes)
{
IndexInput data = (IndexInput)this.Data.Clone();
MonotonicBlockPackedReader addresses = GetAddressInstance(data, field, bytes);
return new LongBinaryDocValuesAnonymousInnerClassHelper2(this, bytes, data, addresses);
}
private BinaryDocValues GetFixedBinary(FieldInfo field, BinaryEntry bytes)
{
IndexInput data = (IndexInput)this.Data.Clone();
return new LongBinaryDocValuesAnonymousInnerClassHelper(this, bytes, data);
}
private BinaryDocValues GetCompressedBinary(FieldInfo field, BinaryEntry bytes)
{
IndexInput data = (IndexInput)this.Data.Clone();
MonotonicBlockPackedReader addresses = GetIntervalInstance(data, field, bytes);
return new CompressedBinaryDocValues(bytes, addresses, data);
}
private BinaryEntry ReadBinaryEntry(IndexInput meta)
{
var entry = new BinaryEntry();
entry.offset = meta.ReadLong();
entry.numBytes = meta.ReadInt();
entry.count = meta.ReadInt();
entry.missingOffset = meta.ReadLong();
if (entry.missingOffset != -1)
{
entry.missingBytes = meta.ReadLong();
}
else
{
entry.missingBytes = 0;
}
return entry;
}
protected override MonotonicBlockPackedReader GetIntervalInstance(IndexInput data, FieldInfo field,
BinaryEntry bytes)
{
throw new InvalidOperationException();
}
