private void UpdateInCache(Uri uri, string withFile) { using (var progressHandler = EmulationManager.Instance.ProgressMonitor.Start("Updating cache")) { lock (CacheDirectory) { var index = ReadBinariesIndex(); BinaryEntry entry; var fileId = 0; if (!index.TryGetValue(uri, out entry)) { foreach (var element in index) { fileId = Math.Max(fileId, element.Value.Index) + 1; } } else { fileId = entry.Index; } FileCopier.Copy(withFile, GetBinaryFileName(fileId), true); // checksum will be 'null' if the uri pattern does not contain // checksum/size information TryGetChecksumAndSizeFromUri(uri, out var checksum, out var size); index[uri] = new BinaryEntry(fileId, size, checksum); WriteBinariesIndex(index); } } }
private void ReadFields(IndexInput meta /*, FieldInfos infos // LUCENENET: Not read */) { int fieldNumber = meta.ReadVInt32(); while (fieldNumber != -1) { // check should be: infos.fieldInfo(fieldNumber) != null, which incorporates negative check // but docvalues updates are currently buggy here (loading extra stuff, etc): LUCENE-5616 if (fieldNumber < 0) { // trickier to validate more: because we re-use for norms, because we use multiple entries // for "composite" types like sortedset, etc. throw new Exception("Invalid field number: " + fieldNumber + " (resource=" + meta + ")"); } byte type = meta.ReadByte(); if (type == Lucene45DocValuesFormat.NUMERIC) { numerics[fieldNumber] = ReadNumericEntry(meta); } else if (type == Lucene45DocValuesFormat.BINARY) { BinaryEntry b = ReadBinaryEntry(meta); binaries[fieldNumber] = b; } else if (type == Lucene45DocValuesFormat.SORTED) { ReadSortedField(fieldNumber, meta /*, infos // LUCENENET: Never read */); } else if (type == Lucene45DocValuesFormat.SORTED_SET) { SortedSetEntry ss = ReadSortedSetEntry(meta); sortedSets[fieldNumber] = ss; if (ss.Format == Lucene45DocValuesConsumer.SORTED_SET_WITH_ADDRESSES) { ReadSortedSetFieldWithAddresses(fieldNumber, meta /*, infos // LUCENENET: Never read */); } else if (ss.Format == Lucene45DocValuesConsumer.SORTED_SET_SINGLE_VALUED_SORTED) { if (meta.ReadVInt32() != fieldNumber) { throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } if (meta.ReadByte() != Lucene45DocValuesFormat.SORTED) { throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } ReadSortedField(fieldNumber, meta /*, infos // LUCENENET: Never read */); } else { throw new Exception(); } } else { throw new Exception("invalid type: " + type + ", resource=" + meta); } fieldNumber = meta.ReadVInt32(); } }
protected override MonotonicBlockPackedReader GetAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) { data.Seek(bytes.AddressesOffset); return new MonotonicBlockPackedReader((IndexInput)data.Clone(), bytes.PackedIntsVersion, bytes.BlockSize, bytes.Count, true); }
private BinaryDocValues LoadBinary(FieldInfo field) { BinaryEntry entry = binaries[field.Number]; data.Seek(entry.offset); var bytes = new PagedBytes(16); bytes.Copy(data, entry.numBytes); var bytesReader = bytes.Freeze(true); if (entry.minLength == entry.maxLength) { int fixedLength = entry.minLength; ramBytesUsed.AddAndGet(bytes.RamBytesUsed()); return(new BinaryDocValuesAnonymousClass(bytesReader, fixedLength)); } else { data.Seek(data.Position + entry.missingBytes); // LUCENENET specific: Renamed from getFilePointer() to match FileStream var addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false); ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + addresses.RamBytesUsed()); return(new BinaryDocValuesAnonymousClass2(bytesReader, addresses)); } }
private bool UpdateInCache(Uri uri, string withFile) { using (var progressHandler = EmulationManager.Instance.ProgressMonitor.Start("Updating cache")) { lock (CacheDirectory) { var index = ReadBinariesIndex(); BinaryEntry entry; var fileId = 0; if (!index.TryGetValue(uri, out entry)) { foreach (var element in index) { fileId = Math.Max(fileId, element.Value.Index) + 1; } } else { fileId = entry.Index; } FileCopier.Copy(withFile, GetBinaryFileName(fileId), true); long size; var checksum = GetChecksumAndSizeFromUri(uri, out size); entry = new BinaryEntry(fileId, size, checksum); if (!Verify(withFile, entry)) { return(false); } index[uri] = entry; WriteBinariesIndex(index); return(true); } } }
private void ReadSortedField(int fieldNumber, IndexInput meta /*, FieldInfos infos // LUCENENET: Never read */) { // sorted = binary + numeric if (meta.ReadVInt32() != fieldNumber) { throw new Exception("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } if (meta.ReadByte() != Lucene45DocValuesFormat.BINARY) { throw new Exception("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } BinaryEntry b = ReadBinaryEntry(meta); binaries[fieldNumber] = b; if (meta.ReadVInt32() != fieldNumber) { throw new Exception("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } if (meta.ReadByte() != Lucene45DocValuesFormat.NUMERIC) { throw new Exception("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } NumericEntry n = ReadNumericEntry(meta); ords[fieldNumber] = n; }
internal static BinaryEntry ReadBinaryEntry(IndexInput meta) { BinaryEntry entry = new BinaryEntry(); entry.format = meta.ReadVInt32(); entry.missingOffset = meta.ReadInt64(); entry.minLength = meta.ReadVInt32(); entry.maxLength = meta.ReadVInt32(); entry.Count = meta.ReadVInt64(); entry.offset = meta.ReadInt64(); switch (entry.format) { case Lucene45DocValuesConsumer.BINARY_FIXED_UNCOMPRESSED: break; case Lucene45DocValuesConsumer.BINARY_PREFIX_COMPRESSED: entry.AddressInterval = meta.ReadVInt32(); entry.AddressesOffset = meta.ReadInt64(); entry.PackedInt32sVersion = meta.ReadVInt32(); entry.BlockSize = meta.ReadVInt32(); break; case Lucene45DocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED: entry.AddressesOffset = meta.ReadInt64(); entry.PackedInt32sVersion = meta.ReadVInt32(); entry.BlockSize = meta.ReadVInt32(); break; default: throw new Exception("Unknown format: " + entry.format + ", input=" + meta); } return(entry); }
protected override MonotonicBlockPackedReader GetAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) { data.Seek(bytes.AddressesOffset); return(new MonotonicBlockPackedReader((IndexInput)data.Clone(), bytes.PackedIntsVersion, bytes.BlockSize, bytes.Count, true)); }
private BinaryDocValues LoadBinary(FieldInfo field) { BinaryEntry entry = binaries[field.Number]; data.Seek(entry.offset); var bytes = new PagedBytes(16); bytes.Copy(data, entry.numBytes); var bytesReader = bytes.Freeze(true); if (entry.minLength == entry.maxLength) { int fixedLength = entry.minLength; ramBytesUsed.AddAndGet(bytes.RamBytesUsed()); return(new BinaryDocValuesAnonymousInnerClassHelper(this, bytesReader, fixedLength)); } else { data.Seek(data.GetFilePointer() + entry.missingBytes); var addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false); ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + addresses.RamBytesUsed()); return(new BinaryDocValuesAnonymousInnerClassHelper2(this, bytesReader, addresses)); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: private index.BinaryDocValues loadBinary(index.FieldInfo field) throws java.io.IOException private BinaryDocValues loadBinary(FieldInfo field) { BinaryEntry entry = binaries[field.number]; data.seek(entry.offset); PagedBytes bytes = new PagedBytes(16); bytes.copy(data, entry.numBytes); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final util.PagedBytes.Reader bytesReader = bytes.freeze(true); PagedBytes.Reader bytesReader = bytes.freeze(true); if (entry.minLength == entry.maxLength) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int fixedLength = entry.minLength; int fixedLength = entry.minLength; ramBytesUsed_Renamed.addAndGet(bytes.ramBytesUsed()); return(new BinaryDocValuesAnonymousInnerClassHelper(this, bytesReader, fixedLength)); } else { data.seek(data.FilePointer + entry.missingBytes); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final util.packed.MonotonicBlockPackedReader addresses = new util.packed.MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false); MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false); ramBytesUsed_Renamed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed()); return(new BinaryDocValuesAnonymousInnerClassHelper2(this, bytesReader, addresses)); } }
private bool Verify(string fileName, BinaryEntry entry) { if (!File.Exists(fileName)) { Logger.LogAs(this, LogLevel.Warning, "Binary {0} found in index but is missing in cache.", fileName); return(false); } if (entry.Checksum != null) { var actualSize = new FileInfo(fileName).Length; if (actualSize != entry.Size) { Logger.LogAs(this, LogLevel.Warning, "Size of the file differs: is {0}B, should be {1}B.", actualSize, entry.Size); return(false); } if (ConfigurationManager.Instance.Get("file-fetcher", "calculate-checksum", true)) { byte[] checksum; using (var progressHandler = EmulationManager.Instance.ProgressMonitor.Start("Calculating SHA1 checksum...")) { checksum = GetSHA1Checksum(fileName); } if (!checksum.SequenceEqual(entry.Checksum)) { Logger.LogAs(this, LogLevel.Warning, "Checksum of the file differs, is {0}, should be {1}.", ChecksumToText(checksum), ChecksumToText(entry.Checksum)); return(false); } } } return(true); }
private BinaryDocValues GetCompressedBinary(FieldInfo field, BinaryEntry bytes) { IndexInput data = (IndexInput)this.data.Clone(); MonotonicBlockPackedReader addresses = GetIntervalInstance(data, field, bytes); return(new CompressedBinaryDocValues(bytes, addresses, data)); }
private BinaryDocValues GetVariableBinary(FieldInfo field, BinaryEntry bytes) { IndexInput data = (IndexInput)this.data.Clone(); MonotonicBlockPackedReader addresses = GetAddressInstance(data, field, bytes); return(new Int64BinaryDocValuesAnonymousInnerClassHelper2(bytes, data, addresses)); }
public CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, IndexInput data) { this.bytes = bytes; this.interval = bytes.AddressInterval; this.addresses = addresses; this.data = data; this.numValues = bytes.Count; this.numIndexValues = addresses.Count; this.termsEnum = GetTermsEnum(data); }
public CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, IndexInput data) { this.Bytes = bytes; this.Interval = bytes.AddressInterval; this.Addresses = addresses; this.Data = data; this.NumValues = bytes.Count; this.NumIndexValues = addresses.Size(); this.TermsEnum_Renamed = GetTermsEnum(data); }
private bool VerifyCachedFile(string fileName, BinaryEntry entry) { if (!File.Exists(fileName)) { Logger.LogAs(this, LogLevel.Warning, "Binary {0} found in index but is missing in cache.", fileName); return(false); } if (entry.Checksum == null) { return(true); } return(VerifySize(fileName, entry.Size) && VerifyChecksum(fileName, entry.Checksum)); }
private static BinaryEntry ReadBinaryEntry(IndexInput meta) // LUCENENET: CA1822: Mark members as static { var entry = new BinaryEntry(); entry.offset = meta.ReadInt64(); entry.numBytes = meta.ReadInt32(); entry.count = meta.ReadInt32(); entry.missingOffset = meta.ReadInt64(); if (entry.missingOffset != -1) { entry.missingBytes = meta.ReadInt64(); } else { entry.missingBytes = 0; } return(entry); }
public override BinaryDocValues GetBinary(FieldInfo field) { BinaryEntry bytes = binaries[field.Number]; switch (bytes.format) { case Lucene45DocValuesConsumer.BINARY_FIXED_UNCOMPRESSED: return(GetFixedBinary(/*field, LUCENENET: Never read */ bytes)); case Lucene45DocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED: return(GetVariableBinary(field, bytes)); case Lucene45DocValuesConsumer.BINARY_PREFIX_COMPRESSED: return(GetCompressedBinary(field, bytes)); default: throw new Exception(); } }
private BinaryEntry ReadBinaryEntry(IndexInput meta) { var entry = new BinaryEntry(); entry.offset = meta.ReadLong(); entry.numBytes = meta.ReadInt(); entry.count = meta.ReadInt(); entry.missingOffset = meta.ReadLong(); if (entry.missingOffset != -1) { entry.missingBytes = meta.ReadLong(); } else { entry.missingBytes = 0; } return(entry); }
private BinaryDocValues LoadBinary(BinaryEntry entry) { data.Seek(entry.offset); var bytes = new byte[entry.numBytes]; data.ReadBytes(bytes, 0, entry.numBytes); data.Seek(entry.offset + entry.numBytes + entry.missingBytes); var address = new int[entry.count + 1]; for (int i = 0; i < entry.count; i++) { address[i] = data.ReadInt(); } address[entry.count] = data.ReadInt(); ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(bytes) + RamUsageEstimator.SizeOf(address)); return(new BinaryDocValuesAnonymousInnerClassHelper(bytes, address)); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: private index.BinaryDocValues loadBinary(BinaryEntry entry) throws java.io.IOException private BinaryDocValues loadBinary(BinaryEntry entry) { data.seek(entry.offset); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final byte[] bytes = new byte[entry.numBytes]; sbyte[] bytes = new sbyte[entry.numBytes]; data.readBytes(bytes, 0, entry.numBytes); data.seek(entry.offset + entry.numBytes + entry.missingBytes); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int[] address = new int[entry.count+1]; int[] address = new int[entry.count + 1]; for (int i = 0; i < entry.count; i++) { address[i] = data.readInt(); } address[entry.count] = data.readInt(); ramBytesUsed_Renamed.addAndGet(RamUsageEstimator.sizeOf(bytes) + RamUsageEstimator.sizeOf(address)); return(new BinaryDocValuesAnonymousInnerClassHelper(this, bytes, address)); }
private BinaryDocValues LoadBinary(FieldInfo field) { BinaryEntry entry = binaries[field.Number]; data.Seek(entry.Offset); PagedBytes bytes = new PagedBytes(16); bytes.Copy(data, entry.NumBytes); PagedBytes.Reader bytesReader = bytes.Freeze(true); if (entry.MinLength == entry.MaxLength) { int fixedLength = entry.MinLength; ramBytesUsed.AddAndGet(bytes.RamBytesUsed()); return(new BinaryDocValuesAnonymousInnerClassHelper(bytesReader, fixedLength)); } else { MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.PackedInt32sVersion, entry.BlockSize, maxDoc, false); ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + addresses.RamBytesUsed()); return(new BinaryDocValuesAnonymousInnerClassHelper2(bytesReader, addresses)); } }
private void ReadSortedSetFieldWithAddresses(int fieldNumber, IndexInput meta, FieldInfos infos) { // sortedset = binary + numeric (addresses) + ordIndex if (meta.ReadVInt32() != fieldNumber) { throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } if (meta.ReadByte() != Lucene45DocValuesFormat.BINARY) { throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } BinaryEntry b = ReadBinaryEntry(meta); binaries[fieldNumber] = b; if (meta.ReadVInt32() != fieldNumber) { throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } if (meta.ReadByte() != Lucene45DocValuesFormat.NUMERIC) { throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } NumericEntry n1 = ReadNumericEntry(meta); ords[fieldNumber] = n1; if (meta.ReadVInt32() != fieldNumber) { throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } if (meta.ReadByte() != Lucene45DocValuesFormat.NUMERIC) { throw new Exception("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")"); } NumericEntry n2 = ReadNumericEntry(meta); ordIndexes[fieldNumber] = n2; }
public override Bits GetDocsWithField(FieldInfo field) { switch (field.DocValuesType) { case FieldInfo.DocValuesType_e.SORTED_SET: return(DocValues.DocsWithValue(GetSortedSet(field), maxDoc)); case FieldInfo.DocValuesType_e.SORTED: return(DocValues.DocsWithValue(GetSorted(field), maxDoc)); case FieldInfo.DocValuesType_e.BINARY: BinaryEntry be = binaries[field.Number]; return(GetMissingBits(field.Number, be.missingOffset, be.missingBytes)); case FieldInfo.DocValuesType_e.NUMERIC: NumericEntry ne = numerics[field.Number]; return(GetMissingBits(field.Number, ne.missingOffset, ne.missingBytes)); default: throw new System.ArgumentOutOfRangeException(); } }
public override IBits GetDocsWithField(FieldInfo field) { switch (field.DocValuesType) { case DocValuesType.SORTED_SET: return(DocValues.DocsWithValue(GetSortedSet(field), maxDoc)); case DocValuesType.SORTED: return(DocValues.DocsWithValue(GetSorted(field), maxDoc)); case DocValuesType.BINARY: BinaryEntry be = binaries[field.Number]; return(GetMissingBits(be.missingOffset)); case DocValuesType.NUMERIC: NumericEntry ne = numerics[field.Number]; return(GetMissingBits(ne.missingOffset)); default: throw new InvalidOperationException(); } }
public override IBits GetDocsWithField(FieldInfo field) { switch (field.DocValuesType) { case DocValuesType.SORTED_SET: return(DocValues.DocsWithValue(GetSortedSet(field), maxDoc)); case DocValuesType.SORTED: return(DocValues.DocsWithValue(GetSorted(field), maxDoc)); case DocValuesType.BINARY: BinaryEntry be = binaries[field.Number]; return(GetMissingBits(field.Number, be.missingOffset, be.missingBytes)); case DocValuesType.NUMERIC: NumericEntry ne = numerics[field.Number]; return(GetMissingBits(field.Number, ne.missingOffset, ne.missingBytes)); default: throw AssertionError.Create(); } }
private BinaryDocValues GetFixedBinary(/* FieldInfo field, // LUCENENET: Never read */ BinaryEntry bytes) { IndexInput data = (IndexInput)this.data.Clone(); return(new Int64BinaryDocValuesAnonymousInnerClassHelper(bytes, data)); }
private bool UpdateInCache(Uri uri, string withFile) { using(var progressHandler = EmulationManager.Instance.ProgressMonitor.Start("Updating cache")) { lock(CacheDirectory) { var index = ReadBinariesIndex(); BinaryEntry entry; var fileId = 0; if(!index.TryGetValue(uri, out entry)) { foreach(var element in index) { fileId = Math.Max(fileId, element.Value.Index) + 1; } } else { fileId = entry.Index; } FileCopier.Copy(withFile, GetBinaryFileName(fileId), true); long size; var checksum = GetChecksumAndSizeFromUri(uri, out size); entry = new BinaryEntry(fileId, size, checksum); if(!Verify(withFile, entry)) { return false; } index[uri] = entry; WriteBinariesIndex(index); return true; } } }
public static Entry CreateEntry( string name, object value, string defaultNamespace, string defaultDeclaringClass ) { string stringValue = value as string; System.Drawing.Bitmap bitmapValue = value as System.Drawing.Bitmap; byte[] rawValue = value as byte[]; Entry entry = null; if(stringValue != null) { entry = new StringEntry( name, stringValue ); } else if(bitmapValue != null) { entry = new BitmapEntry( name, bitmapValue ); } if(rawValue != null) { entry = TinyResourcesEntry.TryCreateTinyResourcesEntry( name, rawValue ); if(entry == null) { entry = new BinaryEntry( name, rawValue ); } } if(entry == null) { throw new Exception(); } if(entry.Namespace.Length == 0) { entry.Namespace = defaultNamespace; } if(entry.ClassName.Length == 0) { ResourceTypeDescription typeDescription = ResourceTypeDescriptionFromResourceType( entry.ResourceType ); entry.ClassName = typeDescription.defaultEnum; if(!string.IsNullOrEmpty( defaultDeclaringClass )) { entry.ClassName = string.Format( "{0}+{1}", defaultDeclaringClass, entry.ClassName ); } } return entry; }
internal static BinaryEntry ReadBinaryEntry(IndexInput meta) { BinaryEntry entry = new BinaryEntry(); entry.Format = meta.ReadVInt(); entry.MissingOffset = meta.ReadLong(); entry.MinLength = meta.ReadVInt(); entry.MaxLength = meta.ReadVInt(); entry.Count = meta.ReadVLong(); entry.Offset = meta.ReadLong(); switch (entry.Format) { case Lucene45DocValuesConsumer.BINARY_FIXED_UNCOMPRESSED: break; case Lucene45DocValuesConsumer.BINARY_PREFIX_COMPRESSED: entry.AddressInterval = meta.ReadVInt(); entry.AddressesOffset = meta.ReadLong(); entry.PackedIntsVersion = meta.ReadVInt(); entry.BlockSize = meta.ReadVInt(); break; case Lucene45DocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED: entry.AddressesOffset = meta.ReadLong(); entry.PackedIntsVersion = meta.ReadVInt(); entry.BlockSize = meta.ReadVInt(); break; default: throw new Exception("Unknown format: " + entry.Format + ", input=" + meta); } return entry; }
/// <summary> /// returns an address instance for prefix-compressed binary values. /// @lucene.internal /// </summary> protected internal virtual MonotonicBlockPackedReader GetIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) { MonotonicBlockPackedReader addresses; long interval = bytes.AddressInterval; lock (AddressInstances) { MonotonicBlockPackedReader addrInstance; if (!AddressInstances.TryGetValue(field.Number, out addrInstance)) { data.Seek(bytes.AddressesOffset); long size; if (bytes.Count % interval == 0) { size = bytes.Count / interval; } else { size = 1L + bytes.Count / interval; } addrInstance = new MonotonicBlockPackedReader(data, bytes.PackedIntsVersion, bytes.BlockSize, size, false); AddressInstances[field.Number] = addrInstance; RamBytesUsed_Renamed.AddAndGet(addrInstance.RamBytesUsed() + RamUsageEstimator.NUM_BYTES_INT); } addresses = addrInstance; } return addresses; }
private BinaryDocValues LoadBinary(BinaryEntry entry) { data.Seek(entry.offset); var bytes = new byte[entry.numBytes]; data.ReadBytes(bytes, 0, entry.numBytes); data.Seek(entry.offset + entry.numBytes + entry.missingBytes); var address = new int[entry.count + 1]; for (int i = 0; i < entry.count; i++) { address[i] = data.ReadInt(); } address[entry.count] = data.ReadInt(); ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(bytes) + RamUsageEstimator.SizeOf(address)); return new BinaryDocValuesAnonymousInnerClassHelper(bytes, address); }
private void ReadFields(IndexInput meta, FieldInfos infos) { int fieldNumber = meta.ReadVInt32(); while (fieldNumber != -1) { // check should be: infos.fieldInfo(fieldNumber) != null, which incorporates negative check // but docvalues updates are currently buggy here (loading extra stuff, etc): LUCENE-5616 if (fieldNumber < 0) { // trickier to validate more: because we re-use for norms, because we use multiple entries // for "composite" types like sortedset, etc. throw new CorruptIndexException("Invalid field number: " + fieldNumber + ", input=" + meta); } int fieldType = meta.ReadByte(); if (fieldType == NUMBER) { var entry = new NumericEntry(); entry.Offset = meta.ReadInt64(); entry.Format = (sbyte)meta.ReadByte(); switch (entry.Format) { case DELTA_COMPRESSED: case TABLE_COMPRESSED: case GCD_COMPRESSED: case UNCOMPRESSED: break; default: throw new CorruptIndexException("Unknown format: " + entry.Format + ", input=" + meta); } if (entry.Format != UNCOMPRESSED) { entry.PackedInt32sVersion = meta.ReadVInt32(); } numerics[fieldNumber] = entry; } else if (fieldType == BYTES) { BinaryEntry entry = new BinaryEntry(); entry.Offset = meta.ReadInt64(); entry.NumBytes = meta.ReadInt64(); entry.MinLength = meta.ReadVInt32(); entry.MaxLength = meta.ReadVInt32(); if (entry.MinLength != entry.MaxLength) { entry.PackedInt32sVersion = meta.ReadVInt32(); entry.BlockSize = meta.ReadVInt32(); } binaries[fieldNumber] = entry; } else if (fieldType == FST) { FSTEntry entry = new FSTEntry(); entry.Offset = meta.ReadInt64(); entry.NumOrds = meta.ReadVInt64(); fsts[fieldNumber] = entry; } else { throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta); } fieldNumber = meta.ReadVInt32(); } }
private void ReadFields(IndexInput meta, FieldInfos infos) { int fieldNumber = meta.ReadVInt(); while (fieldNumber != -1) { int fieldType = meta.ReadByte(); if (fieldType == NUMBER) { var entry = new NumericEntry {offset = meta.ReadLong(), missingOffset = meta.ReadLong()}; if (entry.missingOffset != -1) { entry.missingBytes = meta.ReadLong(); } else { entry.missingBytes = 0; } entry.format = meta.ReadByte(); switch (entry.format) { case DELTA_COMPRESSED: case TABLE_COMPRESSED: case GCD_COMPRESSED: case UNCOMPRESSED: break; default: throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta); } if (entry.format != UNCOMPRESSED) { entry.packedIntsVersion = meta.ReadVInt(); } numerics[fieldNumber] = entry; } else if (fieldType == BYTES) { var entry = new BinaryEntry { offset = meta.ReadLong(), numBytes = meta.ReadLong(), missingOffset = meta.ReadLong() }; if (entry.missingOffset != -1) { entry.missingBytes = meta.ReadLong(); } else { entry.missingBytes = 0; } entry.minLength = meta.ReadVInt(); entry.maxLength = meta.ReadVInt(); if (entry.minLength != entry.maxLength) { entry.packedIntsVersion = meta.ReadVInt(); entry.blockSize = meta.ReadVInt(); } binaries[fieldNumber] = entry; } else if (fieldType == FST) { var entry = new FSTEntry {offset = meta.ReadLong(), numOrds = meta.ReadVLong()}; fsts[fieldNumber] = entry; } else { throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta); } fieldNumber = meta.ReadVInt(); } }
private bool Verify(string fileName, BinaryEntry entry) { if(entry.Checksum != null) { long actualSize = -1; try { actualSize = new FileInfo(fileName).Length; } catch (FileNotFoundException) { Logger.LogAs(this, LogLevel.Warning, "File {0} not found in cached binaries folder.", fileName); return false; } if(actualSize != entry.Size) { Logger.LogAs(this, LogLevel.Warning, "Size of the file differs: is {0}B, should be {1}B.", actualSize, entry.Size); return false; } if(ConfigurationManager.Instance.Get("file-fetcher", "calculate-checksum", true)) { byte[] checksum; using(var progressHandler = EmulationManager.Instance.ProgressMonitor.Start("Calculating SHA1 checksum...")) { checksum = GetSHA1Checksum(fileName); } if(!checksum.SequenceEqual(entry.Checksum)) { Logger.LogAs(this, LogLevel.Warning, "Checksum of the file differs, is {0}, should be {1}.", ChecksumToText(checksum), ChecksumToText(entry.Checksum)); return false; } } } return true; }
private void ReadFields(IndexInput meta, FieldInfos infos) { int fieldNumber = meta.ReadVInt(); while (fieldNumber != -1) { // check should be: infos.fieldInfo(fieldNumber) != null, which incorporates negative check // but docvalues updates are currently buggy here (loading extra stuff, etc): LUCENE-5616 if (fieldNumber < 0) { // trickier to validate more: because we re-use for norms, because we use multiple entries // for "composite" types like sortedset, etc. throw new CorruptIndexException("Invalid field number: " + fieldNumber + ", input=" + meta); } int fieldType = meta.ReadByte(); if (fieldType == NUMBER) { var entry = new NumericEntry {Offset = meta.ReadLong(), Format = (sbyte)meta.ReadByte()}; switch (entry.Format) { case DELTA_COMPRESSED: case TABLE_COMPRESSED: case GCD_COMPRESSED: case UNCOMPRESSED: break; default: throw new CorruptIndexException("Unknown format: " + entry.Format + ", input=" + meta); } if (entry.Format != UNCOMPRESSED) { entry.PackedIntsVersion = meta.ReadVInt(); } Numerics[fieldNumber] = entry; } else if (fieldType == BYTES) { BinaryEntry entry = new BinaryEntry(); entry.Offset = meta.ReadLong(); entry.NumBytes = meta.ReadLong(); entry.MinLength = meta.ReadVInt(); entry.MaxLength = meta.ReadVInt(); if (entry.MinLength != entry.MaxLength) { entry.PackedIntsVersion = meta.ReadVInt(); entry.BlockSize = meta.ReadVInt(); } Binaries[fieldNumber] = entry; } else if (fieldType == FST) { FSTEntry entry = new FSTEntry(); entry.Offset = meta.ReadLong(); entry.NumOrds = meta.ReadVLong(); Fsts[fieldNumber] = entry; } else { throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta); } fieldNumber = meta.ReadVInt(); } }
/// <summary> /// Returns an address instance for variable-length binary values. /// <para/> /// @lucene.internal /// </summary> protected virtual MonotonicBlockPackedReader GetAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) { MonotonicBlockPackedReader addresses; lock (addressInstances) { MonotonicBlockPackedReader addrInstance; if (!addressInstances.TryGetValue(field.Number, out addrInstance) || addrInstance == null) { data.Seek(bytes.AddressesOffset); addrInstance = new MonotonicBlockPackedReader(data, bytes.PackedInt32sVersion, bytes.BlockSize, bytes.Count, false); addressInstances[field.Number] = addrInstance; ramBytesUsed.AddAndGet(addrInstance.RamBytesUsed() + RamUsageEstimator.NUM_BYTES_INT32); } addresses = addrInstance; } return(addresses); }
private BinaryDocValues GetFixedBinary(FieldInfo field, BinaryEntry bytes) { IndexInput data = (IndexInput)this.data.Clone(); return(new Int64BinaryDocValuesAnonymousInnerClassHelper(this, bytes, data)); }
/// <summary> /// Returns an address instance for prefix-compressed binary values. /// <para/> /// @lucene.internal /// </summary> protected virtual MonotonicBlockPackedReader GetIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) { MonotonicBlockPackedReader addresses; long interval = bytes.AddressInterval; lock (addressInstances) { if (!addressInstances.TryGetValue(field.Number, out MonotonicBlockPackedReader addrInstance)) { data.Seek(bytes.AddressesOffset); long size; if (bytes.Count % interval == 0) { size = bytes.Count / interval; } else { size = 1L + bytes.Count / interval; } addrInstance = new MonotonicBlockPackedReader(data, bytes.PackedInt32sVersion, bytes.BlockSize, size, false); addressInstances[field.Number] = addrInstance; ramBytesUsed.AddAndGet(addrInstance.RamBytesUsed() + RamUsageEstimator.NUM_BYTES_INT32); } addresses = addrInstance; } return(addresses); }
private BinaryDocValues GetVariableBinary(FieldInfo field, BinaryEntry bytes) { IndexInput data = (IndexInput)this.Data.Clone(); MonotonicBlockPackedReader addresses = GetAddressInstance(data, field, bytes); return new LongBinaryDocValuesAnonymousInnerClassHelper2(this, bytes, data, addresses); }
private BinaryDocValues GetFixedBinary(FieldInfo field, BinaryEntry bytes) { IndexInput data = (IndexInput)this.Data.Clone(); return new LongBinaryDocValuesAnonymousInnerClassHelper(this, bytes, data); }
private BinaryDocValues GetCompressedBinary(FieldInfo field, BinaryEntry bytes) { IndexInput data = (IndexInput)this.Data.Clone(); MonotonicBlockPackedReader addresses = GetIntervalInstance(data, field, bytes); return new CompressedBinaryDocValues(bytes, addresses, data); }
private BinaryEntry ReadBinaryEntry(IndexInput meta) { var entry = new BinaryEntry(); entry.offset = meta.ReadLong(); entry.numBytes = meta.ReadInt(); entry.count = meta.ReadInt(); entry.missingOffset = meta.ReadLong(); if (entry.missingOffset != -1) { entry.missingBytes = meta.ReadLong(); } else { entry.missingBytes = 0; } return entry; }
protected override MonotonicBlockPackedReader GetIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) { throw new InvalidOperationException(); }