private Range LocateRecordRange(IndexEntryKey key, out IndexEntryKey lowKey, out IndexEntryKey highKey)
{
lowKey = new IndexEntryKey(ulong.MaxValue, long.MaxValue);
highKey = new IndexEntryKey(ulong.MinValue, long.MinValue);
ReadOnlySpan <Midpoint> midpoints = null;
if (_midpoints != null)
{
midpoints = _midpoints.AsSpan();
}
if (midpoints == null)
{
return(new Range(0, Count - 1));
}
long lowerMidpoint = LowerMidpointBound(midpoints, key);
long upperMidpoint = UpperMidpointBound(midpoints, key);
lowKey = midpoints[(int)lowerMidpoint].Key;
highKey = midpoints[(int)upperMidpoint].Key;
return(new Range(midpoints[(int)lowerMidpoint].ItemIndex, midpoints[(int)upperMidpoint].ItemIndex));
}
public IEnumerable <IndexEntry> GetRange(ulong stream, long startNumber, long endNumber, int?limit = null)
{
Ensure.Nonnegative(startNumber, "startNumber");
Ensure.Nonnegative(endNumber, "endNumber");
ulong hash = GetHash(stream);
var result = new List <IndexEntry>();
var startKey = BuildKey(hash, startNumber);
var endKey = BuildKey(hash, endNumber);
if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry))
{
return(result);
}
var workItem = GetWorkItem();
try {
IndexEntryKey lowBoundsCheck, highBoundsCheck;
var recordRange = LocateRecordRange(endKey, out lowBoundsCheck, out highBoundsCheck);
long low = recordRange.Lower;
long high = recordRange.Upper;
while (low < high)
{
var mid = low + (high - low) / 2;
IndexEntry midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version);
var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version);
if (midpointKey.GreaterThan(lowBoundsCheck))
{
throw new MaybeCorruptIndexException(String.Format(
"Midpoint key (stream: {0}, version: {1}) > low bounds check key (stream: {2}, version: {3})",
midpointKey.Stream, midpointKey.Version, lowBoundsCheck.Stream, lowBoundsCheck.Version));
}
else if (!midpointKey.GreaterEqualsThan(highBoundsCheck))
{
throw new MaybeCorruptIndexException(String.Format(
"Midpoint key (stream: {0}, version: {1}) < high bounds check key (stream: {2}, version: {3})",
midpointKey.Stream, midpointKey.Version, highBoundsCheck.Stream, highBoundsCheck.Version));
}
if (midpointKey.SmallerEqualsThan(endKey))
{
high = mid;
highBoundsCheck = midpointKey;
}
else
{
low = mid + 1;
lowBoundsCheck = midpointKey;
}
}
PositionAtEntry(_indexEntrySize, high, workItem);
for (long i = high, n = Count; i < n; ++i)
{
IndexEntry entry = ReadNextNoSeek(workItem, _version);
var candidateKey = new IndexEntryKey(entry.Stream, entry.Version);
if (candidateKey.GreaterThan(endKey))
{
throw new MaybeCorruptIndexException(string.Format(
"entry ({0}@{1}) > endKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.",
entry.Stream, entry.Version, startKey, stream, startNumber, endNumber, Filename));
}
if (candidateKey.SmallerThan(startKey))
{
return(result);
}
result.Add(entry);
if (result.Count == limit)
{
break;
}
}
return(result);
} finally {
ReturnWorkItem(workItem);
}
}
private bool TryGetSmallestEntry(ulong stream, long startNumber, long endNumber, out IndexEntry entry)
{
Ensure.Nonnegative(startNumber, "startNumber");
Ensure.Nonnegative(endNumber, "endNumber");
entry = TableIndex.InvalidIndexEntry;
var startKey = BuildKey(stream, startNumber);
var endKey = BuildKey(stream, endNumber);
if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry))
{
return(false);
}
var workItem = GetWorkItem();
try {
IndexEntryKey lowBoundsCheck, highBoundsCheck;
var recordRange = LocateRecordRange(startKey, out lowBoundsCheck, out highBoundsCheck);
long low = recordRange.Lower;
long high = recordRange.Upper;
while (low < high)
{
var mid = low + (high - low + 1) / 2;
var midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version);
var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version);
if (midpointKey.GreaterThan(lowBoundsCheck))
{
throw new MaybeCorruptIndexException(String.Format(
"Midpoint key (stream: {0}, version: {1}) > low bounds check key (stream: {2}, version: {3})",
midpointKey.Stream, midpointKey.Version, lowBoundsCheck.Stream, lowBoundsCheck.Version));
}
else if (!midpointKey.GreaterEqualsThan(highBoundsCheck))
{
throw new MaybeCorruptIndexException(String.Format(
"Midpoint key (stream: {0}, version: {1}) < high bounds check key (stream: {2}, version: {3})",
midpointKey.Stream, midpointKey.Version, highBoundsCheck.Stream, highBoundsCheck.Version));
}
if (midpointKey.SmallerThan(startKey))
{
high = mid - 1;
highBoundsCheck = midpointKey;
}
else
{
low = mid;
lowBoundsCheck = midpointKey;
}
}
var candEntry = ReadEntry(_indexEntrySize, high, workItem, _version);
var candidateKey = new IndexEntryKey(candEntry.Stream, candEntry.Version);
if (candidateKey.SmallerThan(startKey))
{
throw new MaybeCorruptIndexException(string.Format(
"candEntry ({0}@{1}) < startKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.",
candEntry.Stream, candEntry.Version, startKey, stream, startNumber, endNumber, Filename));
}
if (candidateKey.GreaterThan(endKey))
{
return(false);
}
entry = candEntry;
return(true);
} finally {
ReturnWorkItem(workItem);
}
}
internal Midpoint[] CacheMidpointsAndVerifyHash(int depth, bool skipIndexVerify)
{
var buffer = new byte[4096];
if (depth < 0 || depth > 30)
{
throw new ArgumentOutOfRangeException("depth");
}
var count = Count;
if (count == 0 || depth == 0)
{
return(null);
}
if (skipIndexVerify)
{
Log.Debug("Disabling Verification of PTable");
}
Stream stream = null;
WorkItem workItem = null;
if (Runtime.IsUnixOrMac)
{
workItem = GetWorkItem();
stream = workItem.Stream;
}
else
{
stream = UnbufferedFileStream.Create(_filename, FileMode.Open, FileAccess.Read, FileShare.Read, false,
4096, 4096, false, 4096);
}
try {
int midpointsCount;
Midpoint[] midpoints;
using (MD5 md5 = MD5.Create()) {
try {
midpointsCount = (int)Math.Max(2L, Math.Min((long)1 << depth, count));
midpoints = new Midpoint[midpointsCount];
} catch (OutOfMemoryException exc) {
throw new PossibleToHandleOutOfMemoryException("Failed to allocate memory for Midpoint cache.",
exc);
}
if (skipIndexVerify && (_version >= PTableVersions.IndexV4))
{
if (_midpointsCached == midpointsCount)
{
//index verification is disabled and cached midpoints with the same depth requested are available
//so, we can load them directly from the PTable file
Log.Debug("Loading {midpointsCached} cached midpoints from PTable", _midpointsCached);
long startOffset = stream.Length - MD5Size - PTableFooter.GetSize(_version) -
_midpointsCacheSize;
stream.Seek(startOffset, SeekOrigin.Begin);
for (uint k = 0; k < _midpointsCached; k++)
{
stream.Read(buffer, 0, _indexEntrySize);
IndexEntryKey key;
long index;
if (_version == PTableVersions.IndexV4)
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 8),
BitConverter.ToInt64(buffer, 0));
index = BitConverter.ToInt64(buffer, 8 + 8);
}
else
{
throw new InvalidOperationException("Unknown PTable version: " + _version);
}
midpoints[k] = new Midpoint(key, index);
if (k > 0)
{
if (midpoints[k].Key.GreaterThan(midpoints[k - 1].Key))
{
throw new CorruptIndexException(String.Format(
"Index entry key for midpoint {0} (stream: {1}, version: {2}) < index entry key for midpoint {3} (stream: {4}, version: {5})",
k - 1, midpoints[k - 1].Key.Stream, midpoints[k - 1].Key.Version, k,
midpoints[k].Key.Stream, midpoints[k].Key.Version));
}
else if (midpoints[k - 1].ItemIndex > midpoints[k].ItemIndex)
{
throw new CorruptIndexException(String.Format(
"Item index for midpoint {0} ({1}) > Item index for midpoint {2} ({3})",
k - 1, midpoints[k - 1].ItemIndex, k, midpoints[k].ItemIndex));
}
}
}
return(midpoints);
}
else
{
Log.Debug(
"Skipping loading of cached midpoints from PTable due to count mismatch, cached midpoints: {midpointsCached} / required midpoints: {midpointsCount}",
_midpointsCached, midpointsCount);
}
}
if (!skipIndexVerify)
{
stream.Seek(0, SeekOrigin.Begin);
stream.Read(buffer, 0, PTableHeader.Size);
md5.TransformBlock(buffer, 0, PTableHeader.Size, null, 0);
}
long previousNextIndex = long.MinValue;
var previousKey = new IndexEntryKey(long.MaxValue, long.MaxValue);
for (long k = 0; k < midpointsCount; ++k)
{
long nextIndex = GetMidpointIndex(k, count, midpointsCount);
if (previousNextIndex != nextIndex)
{
if (!skipIndexVerify)
{
ReadUntilWithMd5(PTableHeader.Size + _indexEntrySize * nextIndex, stream, md5);
stream.Read(buffer, 0, _indexKeySize);
md5.TransformBlock(buffer, 0, _indexKeySize, null, 0);
}
else
{
stream.Seek(PTableHeader.Size + _indexEntrySize * nextIndex, SeekOrigin.Begin);
stream.Read(buffer, 0, _indexKeySize);
}
IndexEntryKey key;
if (_version == PTableVersions.IndexV1)
{
key = new IndexEntryKey(BitConverter.ToUInt32(buffer, 4),
BitConverter.ToInt32(buffer, 0));
}
else if (_version == PTableVersions.IndexV2)
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 4),
BitConverter.ToInt32(buffer, 0));
}
else
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 8),
BitConverter.ToInt64(buffer, 0));
}
midpoints[k] = new Midpoint(key, nextIndex);
previousNextIndex = nextIndex;
previousKey = key;
}
else
{
midpoints[k] = new Midpoint(previousKey, previousNextIndex);
}
if (k > 0)
{
if (midpoints[k].Key.GreaterThan(midpoints[k - 1].Key))
{
throw new CorruptIndexException(String.Format(
"Index entry key for midpoint {0} (stream: {1}, version: {2}) < index entry key for midpoint {3} (stream: {4}, version: {5})",
k - 1, midpoints[k - 1].Key.Stream, midpoints[k - 1].Key.Version, k,
midpoints[k].Key.Stream, midpoints[k].Key.Version));
}
else if (midpoints[k - 1].ItemIndex > midpoints[k].ItemIndex)
{
throw new CorruptIndexException(String.Format(
"Item index for midpoint {0} ({1}) > Item index for midpoint {2} ({3})", k - 1,
midpoints[k - 1].ItemIndex, k, midpoints[k].ItemIndex));
}
}
}
if (!skipIndexVerify)
{
ReadUntilWithMd5(stream.Length - MD5Size, stream, md5);
//verify hash (should be at stream.length - MD5Size)
md5.TransformFinalBlock(Empty.ByteArray, 0, 0);
var fileHash = new byte[MD5Size];
stream.Read(fileHash, 0, MD5Size);
ValidateHash(md5.Hash, fileHash);
}
return(midpoints);
}
} catch {
Dispose();
throw;
} finally {
if (Runtime.IsUnixOrMac)
{
if (workItem != null)
{
ReturnWorkItem(workItem);
}
}
else
{
if (stream != null)
{
stream.Dispose();
}
}
}
}
private PTable(string filename,
Guid id,
int initialReaders = ESConsts.PTableInitialReaderCount,
int maxReaders = ESConsts.PTableMaxReaderCount,
int depth = 16)
{
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.NotEmptyGuid(id, "id");
Ensure.Positive(maxReaders, "maxReaders");
Ensure.Nonnegative(depth, "depth");
if (!File.Exists(filename))
{
throw new CorruptIndexException(new PTableNotFoundException(filename));
}
_id = id;
_filename = filename;
Log.Trace("Loading and Verification of PTable '{0}' started...", Path.GetFileName(Filename));
var sw = Stopwatch.StartNew();
_size = new FileInfo(_filename).Length;
File.SetAttributes(_filename, FileAttributes.ReadOnly | FileAttributes.NotContentIndexed);
_workItems = new ObjectPool <WorkItem>(string.Format("PTable {0} work items", _id),
initialReaders,
maxReaders,
() => new WorkItem(filename, DefaultBufferSize),
workItem => workItem.Dispose(),
pool => OnAllWorkItemsDisposed());
var readerWorkItem = GetWorkItem();
try
{
readerWorkItem.Stream.Seek(0, SeekOrigin.Begin);
var header = PTableHeader.FromStream(readerWorkItem.Stream);
if ((header.Version != PTableVersions.Index32Bit) &&
(header.Version != PTableVersions.Index64Bit))
{
throw new CorruptIndexException(new WrongFileVersionException(_filename, header.Version, Version));
}
_version = header.Version;
if (_version == PTableVersions.Index32Bit)
{
_indexEntrySize = IndexEntry32Size;
_indexKeySize = IndexKey32Size;
}
if (_version == PTableVersions.Index64Bit)
{
_indexEntrySize = IndexEntry64Size;
_indexKeySize = IndexKey64Size;
}
_count = ((_size - PTableHeader.Size - MD5Size) / _indexEntrySize);
if (Count == 0)
{
_minEntry = new IndexEntryKey(ulong.MaxValue, int.MaxValue);
_maxEntry = new IndexEntryKey(ulong.MinValue, int.MinValue);
}
else
{
var minEntry = ReadEntry(_indexEntrySize, Count - 1, readerWorkItem, _version);
_minEntry = new IndexEntryKey(minEntry.Stream, minEntry.Version);
var maxEntry = ReadEntry(_indexEntrySize, 0, readerWorkItem, _version);
_maxEntry = new IndexEntryKey(maxEntry.Stream, maxEntry.Version);
}
}
catch (Exception)
{
Dispose();
throw;
}
finally
{
ReturnWorkItem(readerWorkItem);
}
int calcdepth = 0;
try
{
calcdepth = GetDepth(_size, depth);
_midpoints = CacheMidpointsAndVerifyHash(calcdepth);
}
catch (PossibleToHandleOutOfMemoryException)
{
Log.Error("Unable to create midpoints for PTable '{0}' ({1} entries, depth {2} requested). "
+ "Performance hit will occur. OOM Exception.", Path.GetFileName(Filename), Count, depth);
}
Log.Trace("Loading PTable (Version: {0}) '{1}' ({2} entries, cache depth {3}) done in {4}.",
_version, Path.GetFileName(Filename), Count, calcdepth, sw.Elapsed);
}
internal Midpoint[] CacheMidpointsAndVerifyHash(int depth)
{
var buffer = new byte[4096];
if (depth < 0 || depth > 30)
{
throw new ArgumentOutOfRangeException("depth");
}
var count = Count;
if (count == 0 || depth == 0)
{
return(null);
}
#if __MonoCS__
var workItem = GetWorkItem();
var stream = workItem.Stream;
try {
#else
using (var stream = UnbufferedFileStream.Create(_filename, FileMode.Open, FileAccess.Read, FileShare.Read, false, 4096, 4096, false, 4096))
{
#endif
try {
int midpointsCount;
Midpoint[] midpoints;
using (MD5 md5 = MD5.Create())
{
try
{
midpointsCount = (int)Math.Max(2L, Math.Min((long)1 << depth, count));
midpoints = new Midpoint[midpointsCount];
}
catch (OutOfMemoryException exc)
{
throw new PossibleToHandleOutOfMemoryException("Failed to allocate memory for Midpoint cache.", exc);
}
stream.Seek(0, SeekOrigin.Begin);
stream.Read(buffer, 0, PTableHeader.Size);
md5.TransformBlock(buffer, 0, PTableHeader.Size, null, 0);
long previousNextIndex = long.MinValue;
var previousKey = new IndexEntryKey(long.MaxValue, int.MaxValue);
for (long k = 0; k < midpointsCount; ++k)
{
var nextIndex = (long)k * (count - 1) / (midpointsCount - 1);
if (previousNextIndex != nextIndex)
{
ReadUntilWithMd5(PTableHeader.Size + _indexEntrySize * nextIndex, stream, md5);
stream.Read(buffer, 0, _indexKeySize);
md5.TransformBlock(buffer, 0, _indexKeySize, null, 0);
IndexEntryKey key;
if (_version == PTableVersions.Index32Bit)
{
key = new IndexEntryKey(BitConverter.ToUInt32(buffer, 4), BitConverter.ToInt32(buffer, 0));
}
else
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 4), BitConverter.ToInt32(buffer, 0));
}
midpoints[k] = new Midpoint(key, nextIndex);
previousNextIndex = nextIndex;
previousKey = key;
}
else
{
midpoints[k] = new Midpoint(previousKey, previousNextIndex);
}
}
ReadUntilWithMd5(stream.Length - MD5Size, stream, md5);
//verify hash (should be at stream.length - MD5Size)
md5.TransformFinalBlock(Empty.ByteArray, 0, 0);
var fileHash = new byte[MD5Size];
stream.Read(fileHash, 0, MD5Size);
ValidateHash(md5.Hash, fileHash);
return(midpoints);
}
}
catch
{
Dispose();
throw;
}
}
#if __MonoCS__
finally
{
ReturnWorkItem(workItem);
}
#endif
}
public bool SmallerEqualsThan(IndexEntryKey other)
{
if(Stream == other.Stream)
{
return Version <= other.Version;
}
return Stream <= other.Stream;
}
internal Midpoint[] CacheMidpointsAndVerifyHash(int depth)
{
var buffer = new byte[4096];
if (depth < 0 || depth > 30)
throw new ArgumentOutOfRangeException("depth");
var count = Count;
if (count == 0 || depth == 0)
return null;
#if __MonoCS__
var workItem = GetWorkItem();
var stream = workItem.Stream;
try {
#else
using (var stream = UnbufferedFileStream.Create(_filename, FileMode.Open, FileAccess.Read, FileShare.Read, false, 4096, 4096, false, 4096))
{
#endif
try {
int midpointsCount;
Midpoint[] midpoints;
using (MD5 md5 = MD5.Create())
{
try
{
midpointsCount = (int)Math.Max(2L, Math.Min((long)1 << depth, count));
midpoints = new Midpoint[midpointsCount];
}
catch (OutOfMemoryException exc)
{
throw new PossibleToHandleOutOfMemoryException("Failed to allocate memory for Midpoint cache.", exc);
}
stream.Seek(0, SeekOrigin.Begin);
stream.Read(buffer, 0, PTableHeader.Size);
md5.TransformBlock(buffer, 0, PTableHeader.Size, null, 0);
long previousNextIndex = long.MinValue;
var previousKey = new IndexEntryKey(long.MaxValue, int.MaxValue);
for (long k = 0; k < midpointsCount; ++k)
{
var nextIndex = (long)k * (count - 1) / (midpointsCount - 1);
if (previousNextIndex != nextIndex) {
ReadUntilWithMd5(PTableHeader.Size + _indexEntrySize * nextIndex, stream, md5);
stream.Read(buffer, 0, _indexKeySize);
md5.TransformBlock(buffer, 0, _indexKeySize, null, 0);
IndexEntryKey key;
if (_version == PTableVersions.Index32Bit)
{
key = new IndexEntryKey(BitConverter.ToUInt32(buffer, 4), BitConverter.ToInt32(buffer, 0));
}
else
{
key = new IndexEntryKey(BitConverter.ToUInt64(buffer, 4), BitConverter.ToInt32(buffer, 0));
}
midpoints[k] = new Midpoint(key, nextIndex);
previousNextIndex = nextIndex;
previousKey = key;
} else {
midpoints[k] = new Midpoint(previousKey, previousNextIndex);
}
}
ReadUntilWithMd5(stream.Length - MD5Size, stream, md5);
//verify hash (should be at stream.length - MD5Size)
md5.TransformFinalBlock(Empty.ByteArray, 0, 0);
var fileHash = new byte[MD5Size];
stream.Read(fileHash, 0, MD5Size);
ValidateHash(md5.Hash, fileHash);
return midpoints;
}
}
catch
{
Dispose();
throw;
}
}
#if __MonoCS__
finally
{
ReturnWorkItem(workItem);
}
#endif
}
private PTable(string filename,
Guid id,
int initialReaders = ESConsts.PTableInitialReaderCount,
int maxReaders = ESConsts.PTableMaxReaderCount,
int depth = 16)
{
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.NotEmptyGuid(id, "id");
Ensure.Positive(maxReaders, "maxReaders");
Ensure.Nonnegative(depth, "depth");
if (!File.Exists(filename))
throw new CorruptIndexException(new PTableNotFoundException(filename));
_id = id;
_filename = filename;
Log.Trace("Loading and Verification of PTable '{0}' started...", Path.GetFileName(Filename));
var sw = Stopwatch.StartNew();
_size = new FileInfo(_filename).Length;
File.SetAttributes(_filename, FileAttributes.ReadOnly | FileAttributes.NotContentIndexed);
_workItems = new ObjectPool<WorkItem>(string.Format("PTable {0} work items", _id),
initialReaders,
maxReaders,
() => new WorkItem(filename, DefaultBufferSize),
workItem => workItem.Dispose(),
pool => OnAllWorkItemsDisposed());
var readerWorkItem = GetWorkItem();
try
{
readerWorkItem.Stream.Seek(0, SeekOrigin.Begin);
var header = PTableHeader.FromStream(readerWorkItem.Stream);
if ((header.Version != PTableVersions.Index32Bit) &&
(header.Version != PTableVersions.Index64Bit))
throw new CorruptIndexException(new WrongFileVersionException(_filename, header.Version, Version));
_version = header.Version;
if (_version == PTableVersions.Index32Bit)
{
_indexEntrySize = IndexEntry32Size;
_indexKeySize = IndexKey32Size;
}
if (_version == PTableVersions.Index64Bit)
{
_indexEntrySize = IndexEntry64Size;
_indexKeySize = IndexKey64Size;
}
_count = ((_size - PTableHeader.Size - MD5Size) / _indexEntrySize);
if (Count == 0)
{
_minEntry = new IndexEntryKey(ulong.MaxValue, int.MaxValue);
_maxEntry = new IndexEntryKey(ulong.MinValue, int.MinValue);
}
else
{
var minEntry = ReadEntry(_indexEntrySize, Count - 1, readerWorkItem, _version);
_minEntry = new IndexEntryKey(minEntry.Stream, minEntry.Version);
var maxEntry = ReadEntry(_indexEntrySize, 0, readerWorkItem, _version);
_maxEntry = new IndexEntryKey(maxEntry.Stream, maxEntry.Version);
}
}
catch (Exception)
{
Dispose();
throw;
}
finally
{
ReturnWorkItem(readerWorkItem);
}
int calcdepth = 0;
try
{
calcdepth = GetDepth(_size, depth);
_midpoints = CacheMidpointsAndVerifyHash(calcdepth);
}
catch (PossibleToHandleOutOfMemoryException)
{
Log.Error("Unable to create midpoints for PTable '{0}' ({1} entries, depth {2} requested). "
+ "Performance hit will occur. OOM Exception.", Path.GetFileName(Filename), Count, depth);
}
Log.Trace("Loading PTable (Version: {0}) '{1}' ({2} entries, cache depth {3}) done in {4}.",
_version, Path.GetFileName(Filename), Count, calcdepth, sw.Elapsed);
}
public Midpoint(IndexEntryKey key, long itemIndex)
{
Key = key;
ItemIndex = itemIndex;
}
private long UpperMidpointBound(Midpoint[] midpoints, IndexEntryKey key)
{
long l = 0;
long r = midpoints.Length - 1;
while (l < r)
{
long m = l + (r - l) / 2;
if (midpoints[m].Key.SmallerThan(key))
r = m;
else
l = m + 1;
}
return r;
}
private long LowerMidpointBound(Midpoint[] midpoints, IndexEntryKey key)
{
long l = 0;
long r = midpoints.Length - 1;
while (l < r)
{
long m = l + (r - l + 1) / 2;
if (midpoints[m].Key.GreaterThan(key))
l = m;
else
r = m - 1;
}
return l;
}
private Range LocateRecordRange(IndexEntryKey key)
{
var midpoints = _midpoints;
if (midpoints == null)
return new Range(0, Count - 1);
long lowerMidpoint = LowerMidpointBound(midpoints, key);
long upperMidpoint = UpperMidpointBound(midpoints, key);
return new Range(midpoints[lowerMidpoint].ItemIndex, midpoints[upperMidpoint].ItemIndex);
}
public IEnumerable<IndexEntry> GetRange(ulong stream, int startNumber, int endNumber, int? limit = null)
{
Ensure.Nonnegative(startNumber, "startNumber");
Ensure.Nonnegative(endNumber, "endNumber");
ulong hash = GetHash(stream);
var result = new List<IndexEntry>();
var startKey = BuildKey(hash, startNumber);
var endKey = BuildKey(hash, endNumber);
if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry))
return result;
var workItem = GetWorkItem();
try
{
var recordRange = LocateRecordRange(endKey);
long low = recordRange.Lower;
long high = recordRange.Upper;
while (low < high)
{
var mid = low + (high - low) / 2;
IndexEntry midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version);
var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version);
if (midpointKey.SmallerEqualsThan(endKey))
high = mid;
else
low = mid + 1;
}
PositionAtEntry(_indexEntrySize, high, workItem);
for (long i=high, n=Count; i<n; ++i)
{
IndexEntry entry = ReadNextNoSeek(workItem, _version);
var candidateKey = new IndexEntryKey(entry.Stream, entry.Version);
if (candidateKey.GreaterThan(endKey))
throw new Exception(string.Format("entry ({0}@{1}) > endKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.", entry.Stream, entry.Version, startKey, stream, startNumber, endNumber, Filename));
if (candidateKey.SmallerThan(startKey))
return result;
result.Add(entry);
if (result.Count == limit) break;
}
return result;
}
finally
{
ReturnWorkItem(workItem);
}
}
private bool TryGetSmallestEntry(ulong stream, int startNumber, int endNumber, out IndexEntry entry)
{
Ensure.Nonnegative(startNumber, "startNumber");
Ensure.Nonnegative(endNumber, "endNumber");
entry = TableIndex.InvalidIndexEntry;
var startKey = BuildKey(stream, startNumber);
var endKey = BuildKey(stream, endNumber);
if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry))
return false;
var workItem = GetWorkItem();
try
{
var recordRange = LocateRecordRange(startKey);
long low = recordRange.Lower;
long high = recordRange.Upper;
while (low < high)
{
var mid = low + (high - low + 1) / 2;
var midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version);
var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version);
if (midpointKey.SmallerThan(startKey))
high = mid - 1;
else
low = mid;
}
var candEntry = ReadEntry(_indexEntrySize, high, workItem, _version);
var candidateKey = new IndexEntryKey(candEntry.Stream, candEntry.Version);
if (candidateKey.SmallerThan(startKey))
throw new Exception(string.Format("candEntry ({0}@{1}) < startKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.", candEntry.Stream, candEntry.Version, startKey, stream, startNumber, endNumber, Filename));
if (candidateKey.GreaterThan(endKey))
return false;
entry = candEntry;
return true;
}
finally
{
ReturnWorkItem(workItem);
}
}
private PTable(string filename,
Guid id,
int initialReaders,
int maxReaders,
int depth = 16,
bool skipIndexVerify = false)
{
Ensure.NotNullOrEmpty(filename, "filename");
Ensure.NotEmptyGuid(id, "id");
Ensure.Positive(maxReaders, "maxReaders");
Ensure.Nonnegative(depth, "depth");
if (!File.Exists(filename))
{
throw new CorruptIndexException(new PTableNotFoundException(filename));
}
_id = id;
_filename = filename;
Log.Verbose("Loading " + (skipIndexVerify ? "" : "and Verification ") + "of PTable '{pTable}' started...",
Path.GetFileName(Filename));
var sw = Stopwatch.StartNew();
_size = new FileInfo(_filename).Length;
File.SetAttributes(_filename, FileAttributes.ReadOnly | FileAttributes.NotContentIndexed);
_workItems = new ObjectPool <WorkItem>(string.Format("PTable {0} work items", _id),
initialReaders,
maxReaders,
() => new WorkItem(filename, DefaultBufferSize),
workItem => workItem.Dispose(),
pool => OnAllWorkItemsDisposed());
var readerWorkItem = GetWorkItem();
try {
readerWorkItem.Stream.Seek(0, SeekOrigin.Begin);
var header = PTableHeader.FromStream(readerWorkItem.Stream);
if ((header.Version != PTableVersions.IndexV1) &&
(header.Version != PTableVersions.IndexV2) &&
(header.Version != PTableVersions.IndexV3) &&
(header.Version != PTableVersions.IndexV4))
{
throw new CorruptIndexException(new WrongFileVersionException(_filename, header.Version, Version));
}
_version = header.Version;
if (_version == PTableVersions.IndexV1)
{
_indexEntrySize = IndexEntryV1Size;
_indexKeySize = IndexKeyV1Size;
}
if (_version == PTableVersions.IndexV2)
{
_indexEntrySize = IndexEntryV2Size;
_indexKeySize = IndexKeyV2Size;
}
if (_version == PTableVersions.IndexV3)
{
_indexEntrySize = IndexEntryV3Size;
_indexKeySize = IndexKeyV3Size;
}
if (_version >= PTableVersions.IndexV4)
{
//read the PTable footer
var previousPosition = readerWorkItem.Stream.Position;
readerWorkItem.Stream.Seek(readerWorkItem.Stream.Length - MD5Size - PTableFooter.GetSize(_version),
SeekOrigin.Begin);
var footer = PTableFooter.FromStream(readerWorkItem.Stream);
if (footer.Version != header.Version)
{
throw new CorruptIndexException(
String.Format("PTable header/footer version mismatch: {0}/{1}", header.Version,
footer.Version), new InvalidFileException("Invalid PTable file."));
}
if (_version == PTableVersions.IndexV4)
{
_indexEntrySize = IndexEntryV4Size;
_indexKeySize = IndexKeyV4Size;
}
else
{
throw new InvalidOperationException("Unknown PTable version: " + _version);
}
_midpointsCached = footer.NumMidpointsCached;
_midpointsCacheSize = _midpointsCached * _indexEntrySize;
readerWorkItem.Stream.Seek(previousPosition, SeekOrigin.Begin);
}
long indexEntriesTotalSize = (_size - PTableHeader.Size - _midpointsCacheSize -
PTableFooter.GetSize(_version) - MD5Size);
if (indexEntriesTotalSize < 0)
{
throw new CorruptIndexException(String.Format(
"Total size of index entries < 0: {0}. _size: {1}, header size: {2}, _midpointsCacheSize: {3}, footer size: {4}, md5 size: {5}",
indexEntriesTotalSize, _size, PTableHeader.Size, _midpointsCacheSize,
PTableFooter.GetSize(_version), MD5Size));
}
else if (indexEntriesTotalSize % _indexEntrySize != 0)
{
throw new CorruptIndexException(String.Format(
"Total size of index entries: {0} is not divisible by index entry size: {1}",
indexEntriesTotalSize, _indexEntrySize));
}
_count = indexEntriesTotalSize / _indexEntrySize;
if (_version >= PTableVersions.IndexV4 && _count > 0 && _midpointsCached > 0 && _midpointsCached < 2)
{
//if there is at least 1 index entry with version>=4 and there are cached midpoints, there should always be at least 2 midpoints cached
throw new CorruptIndexException(String.Format(
"Less than 2 midpoints cached in PTable. Index entries: {0}, Midpoints cached: {1}", _count,
_midpointsCached));
}
else if (_count >= 2 && _midpointsCached > _count)
{
//if there are at least 2 index entries, midpoints count should be at most the number of index entries
throw new CorruptIndexException(String.Format(
"More midpoints cached in PTable than index entries. Midpoints: {0} , Index entries: {1}",
_midpointsCached, _count));
}
if (Count == 0)
{
_minEntry = new IndexEntryKey(ulong.MaxValue, long.MaxValue);
_maxEntry = new IndexEntryKey(ulong.MinValue, long.MinValue);
}
else
{
var minEntry = ReadEntry(_indexEntrySize, Count - 1, readerWorkItem, _version);
_minEntry = new IndexEntryKey(minEntry.Stream, minEntry.Version);
var maxEntry = ReadEntry(_indexEntrySize, 0, readerWorkItem, _version);
_maxEntry = new IndexEntryKey(maxEntry.Stream, maxEntry.Version);
}
} catch (Exception) {
Dispose();
throw;
} finally {
ReturnWorkItem(readerWorkItem);
}
int calcdepth = 0;
try {
calcdepth = GetDepth(_count * _indexEntrySize, depth);
_midpoints = CacheMidpointsAndVerifyHash(calcdepth, skipIndexVerify);
} catch (PossibleToHandleOutOfMemoryException) {
Log.Error(
"Unable to create midpoints for PTable '{pTable}' ({count} entries, depth {depth} requested). "
+ "Performance hit will occur. OOM Exception.", Path.GetFileName(Filename), Count, depth);
}
Log.Verbose(
"Loading PTable (Version: {version}) '{pTable}' ({count} entries, cache depth {depth}) done in {elapsed}.",
_version, Path.GetFileName(Filename), Count, calcdepth, sw.Elapsed);
}
public Midpoint(IndexEntryKey key, long itemIndex)
{
Key = key;
ItemIndex = itemIndex;
}
public bool GreaterEqualsThan(IndexEntryKey other)
{
if (Stream == other.Stream)
{
return Version >= other.Version;
}
return Stream >= other.Stream;
}
