private bool TryGetSmallestEntry(ulong stream, long startNumber, long endNumber, out IndexEntry entry)
{
Ensure.Nonnegative(startNumber, "startNumber");
Ensure.Nonnegative(endNumber, "endNumber");
entry = TableIndex.InvalidIndexEntry;
var startKey = BuildKey(stream, startNumber);
var endKey = BuildKey(stream, endNumber);
if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry))
{
return(false);
}
var workItem = GetWorkItem();
try {
IndexEntryKey lowBoundsCheck, highBoundsCheck;
var recordRange = LocateRecordRange(startKey, out lowBoundsCheck, out highBoundsCheck);
long low = recordRange.Lower;
long high = recordRange.Upper;
while (low < high)
{
var mid = low + (high - low + 1) / 2;
var midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version);
var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version);
if (midpointKey.GreaterThan(lowBoundsCheck))
{
throw new MaybeCorruptIndexException(String.Format(
"Midpoint key (stream: {0}, version: {1}) > low bounds check key (stream: {2}, version: {3})",
midpointKey.Stream, midpointKey.Version, lowBoundsCheck.Stream, lowBoundsCheck.Version));
}
else if (!midpointKey.GreaterEqualsThan(highBoundsCheck))
{
throw new MaybeCorruptIndexException(String.Format(
"Midpoint key (stream: {0}, version: {1}) < high bounds check key (stream: {2}, version: {3})",
midpointKey.Stream, midpointKey.Version, highBoundsCheck.Stream, highBoundsCheck.Version));
}
if (midpointKey.SmallerThan(startKey))
{
high = mid - 1;
highBoundsCheck = midpointKey;
}
else
{
low = mid;
lowBoundsCheck = midpointKey;
}
}
var candEntry = ReadEntry(_indexEntrySize, high, workItem, _version);
var candidateKey = new IndexEntryKey(candEntry.Stream, candEntry.Version);
if (candidateKey.SmallerThan(startKey))
{
throw new MaybeCorruptIndexException(string.Format(
"candEntry ({0}@{1}) < startKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.",
candEntry.Stream, candEntry.Version, startKey, stream, startNumber, endNumber, Filename));
}
if (candidateKey.GreaterThan(endKey))
{
return(false);
}
entry = candEntry;
return(true);
} finally {
ReturnWorkItem(workItem);
}
}
public IEnumerable <IndexEntry> GetRange(ulong stream, long startNumber, long endNumber, int?limit = null)
{
Ensure.Nonnegative(startNumber, "startNumber");
Ensure.Nonnegative(endNumber, "endNumber");
ulong hash = GetHash(stream);
var result = new List <IndexEntry>();
var startKey = BuildKey(hash, startNumber);
var endKey = BuildKey(hash, endNumber);
if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry))
{
return(result);
}
var workItem = GetWorkItem();
try {
IndexEntryKey lowBoundsCheck, highBoundsCheck;
var recordRange = LocateRecordRange(endKey, out lowBoundsCheck, out highBoundsCheck);
long low = recordRange.Lower;
long high = recordRange.Upper;
while (low < high)
{
var mid = low + (high - low) / 2;
IndexEntry midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version);
var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version);
if (midpointKey.GreaterThan(lowBoundsCheck))
{
throw new MaybeCorruptIndexException(String.Format(
"Midpoint key (stream: {0}, version: {1}) > low bounds check key (stream: {2}, version: {3})",
midpointKey.Stream, midpointKey.Version, lowBoundsCheck.Stream, lowBoundsCheck.Version));
}
else if (!midpointKey.GreaterEqualsThan(highBoundsCheck))
{
throw new MaybeCorruptIndexException(String.Format(
"Midpoint key (stream: {0}, version: {1}) < high bounds check key (stream: {2}, version: {3})",
midpointKey.Stream, midpointKey.Version, highBoundsCheck.Stream, highBoundsCheck.Version));
}
if (midpointKey.SmallerEqualsThan(endKey))
{
high = mid;
highBoundsCheck = midpointKey;
}
else
{
low = mid + 1;
lowBoundsCheck = midpointKey;
}
}
PositionAtEntry(_indexEntrySize, high, workItem);
for (long i = high, n = Count; i < n; ++i)
{
IndexEntry entry = ReadNextNoSeek(workItem, _version);
var candidateKey = new IndexEntryKey(entry.Stream, entry.Version);
if (candidateKey.GreaterThan(endKey))
{
throw new MaybeCorruptIndexException(string.Format(
"entry ({0}@{1}) > endKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.",
entry.Stream, entry.Version, startKey, stream, startNumber, endNumber, Filename));
}
if (candidateKey.SmallerThan(startKey))
{
return(result);
}
result.Add(entry);
if (result.Count == limit)
{
break;
}
}
return(result);
} finally {
ReturnWorkItem(workItem);
}
}
private IEnumerable <IndexEntry> GetRangeInternal(ulong hash, long startVersion, long endVersion,
int?limit = null)
{
if (startVersion < 0)
{
throw new ArgumentOutOfRangeException("startVersion");
}
if (endVersion < 0)
{
throw new ArgumentOutOfRangeException("endVersion");
}
var candidates = new List <IEnumerator <IndexEntry> >();
var awaiting = _awaitingMemTables;
for (int index = 0; index < awaiting.Count; index++)
{
var range = awaiting[index].Table.GetRange(hash, startVersion, endVersion, limit).GetEnumerator();
if (range.MoveNext())
{
candidates.Add(range);
}
}
var map = _indexMap;
foreach (var table in map.InOrder())
{
var range = table.GetRange(hash, startVersion, endVersion, limit).GetEnumerator();
if (range.MoveNext())
{
candidates.Add(range);
}
}
var last = new IndexEntry(0, 0, 0);
var first = true;
var sortedCandidates = new List <IndexEntry>();
while (candidates.Count > 0)
{
var maxIdx = GetMaxOf(candidates);
var winner = candidates[maxIdx];
var best = winner.Current;
if (first || ((last.Stream != best.Stream) && (last.Version != best.Version)) ||
last.Position != best.Position)
{
last = best;
sortedCandidates.Add(best);
first = false;
}
if (!winner.MoveNext())
{
candidates.RemoveAt(maxIdx);
}
}
return(sortedCandidates);
}
public bool TryGetOldestEntry(ulong stream, out IndexEntry entry)
{
ulong hash = GetHash(stream);
return(TryGetSmallestEntry(hash, 0, long.MaxValue, out entry));
}
private static PTable MergeTo2(IList <PTable> tables, long numIndexEntries, int indexEntrySize,
string outputFile,
Func <string, ulong, ulong> upgradeHash, Func <IndexEntry, bool> existsAt,
Func <IndexEntry, Tuple <string, bool> > readRecord,
byte version, int initialReaders, int maxReaders,
int cacheDepth, bool skipIndexVerify)
{
Log.Debug("PTables merge started (specialized for <= 2 tables).");
var watch = Stopwatch.StartNew();
var fileSizeUpToIndexEntries = GetFileSizeUpToIndexEntries(numIndexEntries, version);
var enumerators = tables
.Select(table => new EnumerableTable(version, table, upgradeHash, existsAt, readRecord)).ToList();
try {
long dumpedEntryCount = 0;
using (var f = new FileStream(outputFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None,
DefaultSequentialBufferSize, FileOptions.SequentialScan)) {
f.SetLength(fileSizeUpToIndexEntries);
f.Seek(0, SeekOrigin.Begin);
using (var md5 = MD5.Create())
using (var cs = new CryptoStream(f, md5, CryptoStreamMode.Write))
using (var bs = new BufferedStream(cs, DefaultSequentialBufferSize)) {
// WRITE HEADER
var headerBytes = new PTableHeader(version).AsByteArray();
cs.Write(headerBytes, 0, headerBytes.Length);
// WRITE INDEX ENTRIES
var buffer = new byte[indexEntrySize];
long indexEntry = 0L;
List <Midpoint> midpoints = new List <Midpoint>();
var requiredMidpointCount =
GetRequiredMidpointCountCached(numIndexEntries, version, cacheDepth);
var enum1 = enumerators[0];
var enum2 = enumerators[1];
bool available1 = enum1.MoveNext();
bool available2 = enum2.MoveNext();
IndexEntry current;
while (available1 || available2)
{
var entry1 = new IndexEntry(enum1.Current.Stream, enum1.Current.Version,
enum1.Current.Position);
var entry2 = new IndexEntry(enum2.Current.Stream, enum2.Current.Version,
enum2.Current.Position);
if (available1 && (!available2 || entry1.CompareTo(entry2) > 0))
{
current = entry1;
available1 = enum1.MoveNext();
}
else
{
current = entry2;
available2 = enum2.MoveNext();
}
AppendRecordTo(bs, buffer, version, current, indexEntrySize);
if (version >= PTableVersions.IndexV4 &&
IsMidpointIndex(indexEntry, numIndexEntries, requiredMidpointCount))
{
midpoints.Add(new Midpoint(new IndexEntryKey(current.Stream, current.Version),
indexEntry));
}
indexEntry++;
dumpedEntryCount++;
}
//WRITE MIDPOINTS
if (version >= PTableVersions.IndexV4)
{
if (dumpedEntryCount != numIndexEntries)
{
//if index entries have been removed, compute the midpoints again
numIndexEntries = dumpedEntryCount;
requiredMidpointCount = GetRequiredMidpointCount(numIndexEntries, version, cacheDepth);
midpoints = ComputeMidpoints(bs, f, version, indexEntrySize, numIndexEntries,
requiredMidpointCount, midpoints);
}
WriteMidpointsTo(bs, f, version, indexEntrySize, buffer, dumpedEntryCount, numIndexEntries,
requiredMidpointCount, midpoints);
}
bs.Flush();
cs.FlushFinalBlock();
f.SetLength(f.Position + MD5Size);
// WRITE MD5
var hash = md5.Hash;
f.Write(hash, 0, hash.Length);
f.FlushToDisk();
}
}
Log.Debug(
"PTables merge finished in {elapsed} ([{entryCount}] entries merged into {dumpedEntryCount}).",
watch.Elapsed, string.Join(", ", tables.Select(x => x.Count)), dumpedEntryCount);
return(new PTable(outputFile, Guid.NewGuid(), initialReaders, maxReaders, cacheDepth, skipIndexVerify));
} finally {
foreach (var enumerator in enumerators)
{
enumerator.Dispose();
}
}
}
private static void AppendRecordTo(Stream stream, byte[] buffer, byte version, IndexEntry entry,
int indexEntrySize)
{
var bytes = entry.Bytes;
if (version == PTableVersions.IndexV1)
{
var entryV1 = new IndexEntryV1((uint)entry.Stream, (int)entry.Version, entry.Position);
bytes = entryV1.Bytes;
}
else if (version == PTableVersions.IndexV2)
{
var entryV2 = new IndexEntryV2(entry.Stream, (int)entry.Version, entry.Position);
bytes = entryV2.Bytes;
}
Marshal.Copy((IntPtr)bytes, buffer, 0, indexEntrySize);
stream.Write(buffer, 0, indexEntrySize);
}
private static PTable MergeTo2(IList <PTable> tables, long fileSize, int indexEntrySize, string outputFile,
Func <string, ulong, ulong> upgradeHash, Func <IndexEntry, bool> existsAt, Func <IndexEntry, Tuple <string, bool> > readRecord,
byte version, int cacheDepth)
{
Log.Trace("PTables merge started (specialized for <= 2 tables).");
var watch = Stopwatch.StartNew();
var enumerators = tables.Select(table => new EnumerableTable(version, table, upgradeHash, existsAt, readRecord)).ToList();
long dumpedEntryCount = 0;
using (var f = new FileStream(outputFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None,
DefaultSequentialBufferSize, FileOptions.SequentialScan))
{
f.SetLength(fileSize);
f.Seek(0, SeekOrigin.Begin);
using (var md5 = MD5.Create())
using (var cs = new CryptoStream(f, md5, CryptoStreamMode.Write))
using (var bs = new BufferedStream(cs, DefaultSequentialBufferSize))
{
// WRITE HEADER
var headerBytes = new PTableHeader(version).AsByteArray();
cs.Write(headerBytes, 0, headerBytes.Length);
// WRITE INDEX ENTRIES
var buffer = new byte[indexEntrySize];
var enum1 = enumerators[0];
var enum2 = enumerators[1];
bool available1 = enum1.MoveNext();
bool available2 = enum2.MoveNext();
IndexEntry current;
while (available1 || available2)
{
var entry1 = new IndexEntry(enum1.Current.Stream, enum1.Current.Version, enum1.Current.Position);
var entry2 = new IndexEntry(enum2.Current.Stream, enum2.Current.Version, enum2.Current.Position);
if (available1 && (!available2 || entry1.CompareTo(entry2) > 0))
{
current = entry1;
available1 = enum1.MoveNext();
}
else
{
current = entry2;
available2 = enum2.MoveNext();
}
if (existsAt(current))
{
AppendRecordTo(bs, buffer, version, current, indexEntrySize);
dumpedEntryCount += 1;
}
}
bs.Flush();
cs.FlushFinalBlock();
f.SetLength(f.Position + MD5Size);
// WRITE MD5
var hash = md5.Hash;
f.Write(hash, 0, hash.Length);
f.FlushToDisk();
}
}
Log.Trace("PTables merge finished in {0} ([{1}] entries merged into {2}).",
watch.Elapsed, string.Join(", ", tables.Select(x => x.Count)), dumpedEntryCount);
return(new PTable(outputFile, Guid.NewGuid(), version, depth: cacheDepth));
}
private bool TryGetSmallestEntry(uint stream, int startNumber, int endNumber, out IndexEntry entry)
{
Ensure.Nonnegative(startNumber, "startNumber");
Ensure.Nonnegative(endNumber, "endNumber");
entry = TableIndex.InvalidIndexEntry;
var startKey = BuildKey(stream, startNumber);
var endKey = BuildKey(stream, endNumber);
if (startKey > _maxEntry || endKey < _minEntry)
{
return(false);
}
var workItem = GetWorkItem();
try
{
var recordRange = LocateRecordRange(startKey);
int low = recordRange.Lower;
int high = recordRange.Upper;
while (low < high)
{
var mid = low + (high - low + 1) / 2;
IndexEntry midpoint = ReadEntry(mid, workItem);
if (midpoint.Key < startKey)
{
high = mid - 1;
}
else
{
low = mid;
}
}
var candEntry = ReadEntry(high, workItem);
if (candEntry.Key < startKey)
{
throw new Exception(string.Format("candEntry.Key {0} < startKey {1}, stream {2}, startNum {3}, endNum {4}, PTable: {5}.", candEntry.Key, startKey, stream, startNumber, endNumber, Filename));
}
if (candEntry.Key > endKey)
{
return(false);
}
entry = candEntry;
return(true);
}
finally
{
ReturnWorkItem(workItem);
}
}
public bool TryGetOldestEntry(uint stream, out IndexEntry entry)
{
return(TryGetSmallestEntry(stream, 0, int.MaxValue, out entry));
}
public bool TryGetLatestEntry(uint stream, out IndexEntry entry)
{
return(TryGetLargestEntry(stream, 0, int.MaxValue, out entry));
}
private bool TryGetLargestEntry(ulong stream, long startNumber, long endNumber, out IndexEntry entry)
{
Ensure.Nonnegative(startNumber, "startNumber");
Ensure.Nonnegative(endNumber, "endNumber");
entry = TableIndex.InvalidIndexEntry;
var startKey = BuildKey(stream, startNumber);
var endKey = BuildKey(stream, endNumber);
if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry))
{
return(false);
}
var workItem = GetWorkItem();
try
{
var recordRange = LocateRecordRange(endKey);
long low = recordRange.Lower;
long high = recordRange.Upper;
while (low < high)
{
var mid = low + (high - low) / 2;
IndexEntry midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version);
var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version);
if (midpointKey.GreaterThan(endKey))
{
low = mid + 1;
}
else
{
high = mid;
}
}
var candEntry = ReadEntry(_indexEntrySize, high, workItem, _version);
var candKey = new IndexEntryKey(candEntry.Stream, candEntry.Version);
if (candKey.GreaterThan(endKey))
{
throw new Exception(string.Format("candEntry ({0}@{1}) > startKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.", candEntry.Stream, candEntry.Version, startKey, stream, startNumber, endNumber, Filename));
}
if (candKey.SmallerThan(startKey))
{
return(false);
}
entry = candEntry;
return(true);
}
finally
{
ReturnWorkItem(workItem);
}
}
