internal UnmanagedMemoryAppendOnlyList <Midpoint> CacheMidpointsAndVerifyHash(int depth, bool skipIndexVerify)
{
var buffer = new byte[4096];
if (depth < 0 || depth > 30)
{
throw new ArgumentOutOfRangeException("depth");
}
var count = Count;
if (count == 0 || depth == 0)
{
return(null);
}
if (skipIndexVerify)
{
Log.Debug("Disabling Verification of PTable");
}
Stream stream = null;
WorkItem workItem = null;
if (Runtime.IsUnixOrMac)
{
workItem = GetWorkItem();
stream = workItem.Stream;
}
else
{
stream = UnbufferedFileStream.Create(_filename, FileMode.Open, FileAccess.Read, FileShare.Read, false,
4096, 4096, false, 4096);
}
UnmanagedMemoryAppendOnlyList <Midpoint> midpoints = null;
try {
using (MD5 md5 = MD5.Create()) {
int midpointsCount;
try {
midpointsCount = (int)Math.Max(2L, Math.Min((long)1 << depth, count));
midpoints = new UnmanagedMemoryAppendOnlyList <Midpoint>(midpointsCount);
} catch (OutOfMemoryException exc) {
throw new PossibleToHandleOutOfMemoryException("Failed to allocate memory for Midpoint cache.",
exc);
}
if (skipIndexVerify && (_version >= PTableVersions.IndexV4))
{
if (_midpointsCached == midpointsCount)
{
//index verification is disabled and cached midpoints with the same depth requested are available
//so, we can load them directly from the PTable file
Log.Debug("Loading {midpointsCached} cached midpoints from PTable", _midpointsCached);
long startOffset = stream.Length - MD5Size - PTableFooter.GetSize(_version) -
_midpointsCacheSize;
stream.Seek(startOffset, SeekOrigin.Begin);
for (int k = 0; k < (int)_midpointsCached; k++)
{
stream.Read(buffer, 0, _indexEntrySize);
IndexEntryKey key;
long index;
if (_version == PTableVersions.IndexV4)
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 8),
BitConverter.ToInt64(buffer, 0));
index = BitConverter.ToInt64(buffer, 8 + 8);
}
else
{
throw new InvalidOperationException("Unknown PTable version: " + _version);
}
midpoints.Add(new Midpoint(key, index));
if (k > 0)
{
if (midpoints[k].Key.GreaterThan(midpoints[k - 1].Key))
{
throw new CorruptIndexException(String.Format(
"Index entry key for midpoint {0} (stream: {1}, version: {2}) < index entry key for midpoint {3} (stream: {4}, version: {5})",
k - 1, midpoints[k - 1].Key.Stream, midpoints[k - 1].Key.Version, k,
midpoints[k].Key.Stream, midpoints[k].Key.Version));
}
else if (midpoints[k - 1].ItemIndex > midpoints[k].ItemIndex)
{
throw new CorruptIndexException(String.Format(
"Item index for midpoint {0} ({1}) > Item index for midpoint {2} ({3})",
k - 1, midpoints[k - 1].ItemIndex, k, midpoints[k].ItemIndex));
}
}
}
return(midpoints);
}
else
{
Log.Debug(
"Skipping loading of cached midpoints from PTable due to count mismatch, cached midpoints: {midpointsCached} / required midpoints: {midpointsCount}",
_midpointsCached, midpointsCount);
}
}
if (!skipIndexVerify)
{
stream.Seek(0, SeekOrigin.Begin);
stream.Read(buffer, 0, PTableHeader.Size);
md5.TransformBlock(buffer, 0, PTableHeader.Size, null, 0);
}
long previousNextIndex = long.MinValue;
var previousKey = new IndexEntryKey(long.MaxValue, long.MaxValue);
for (int k = 0; k < midpointsCount; ++k)
{
long nextIndex = GetMidpointIndex(k, count, midpointsCount);
if (previousNextIndex != nextIndex)
{
if (!skipIndexVerify)
{
ReadUntilWithMd5(PTableHeader.Size + _indexEntrySize * nextIndex, stream, md5);
stream.Read(buffer, 0, _indexKeySize);
md5.TransformBlock(buffer, 0, _indexKeySize, null, 0);
}
else
{
stream.Seek(PTableHeader.Size + _indexEntrySize * nextIndex, SeekOrigin.Begin);
stream.Read(buffer, 0, _indexKeySize);
}
IndexEntryKey key;
if (_version == PTableVersions.IndexV1)
{
key = new IndexEntryKey(BitConverter.ToUInt32(buffer, 4),
BitConverter.ToInt32(buffer, 0));
}
else if (_version == PTableVersions.IndexV2)
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 4),
BitConverter.ToInt32(buffer, 0));
}
else
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 8),
BitConverter.ToInt64(buffer, 0));
}
midpoints.Add(new Midpoint(key, nextIndex));
previousNextIndex = nextIndex;
previousKey = key;
}
else
{
midpoints.Add(new Midpoint(previousKey, previousNextIndex));
}
if (k > 0)
{
if (midpoints[k].Key.GreaterThan(midpoints[k - 1].Key))
{
throw new CorruptIndexException(String.Format(
"Index entry key for midpoint {0} (stream: {1}, version: {2}) < index entry key for midpoint {3} (stream: {4}, version: {5})",
k - 1, midpoints[k - 1].Key.Stream, midpoints[k - 1].Key.Version, k,
midpoints[k].Key.Stream, midpoints[k].Key.Version));
}
else if (midpoints[k - 1].ItemIndex > midpoints[k].ItemIndex)
{
throw new CorruptIndexException(String.Format(
"Item index for midpoint {0} ({1}) > Item index for midpoint {2} ({3})", k - 1,
midpoints[k - 1].ItemIndex, k, midpoints[k].ItemIndex));
}
}
}
if (!skipIndexVerify)
{
ReadUntilWithMd5(stream.Length - MD5Size, stream, md5);
//verify hash (should be at stream.length - MD5Size)
md5.TransformFinalBlock(Empty.ByteArray, 0, 0);
var fileHash = new byte[MD5Size];
stream.Read(fileHash, 0, MD5Size);
ValidateHash(md5.Hash, fileHash);
}
return(midpoints);
}
} catch (PossibleToHandleOutOfMemoryException) {
midpoints?.Dispose();
throw;
} catch {
midpoints?.Dispose();
Dispose();
throw;
} finally {
if (Runtime.IsUnixOrMac)
{
if (workItem != null)
{
ReturnWorkItem(workItem);
}
}
else
{
if (stream != null)
{
stream.Dispose();
}
}
}
}
public void TearDown()
{
_list?.Dispose();
}