private bool TryGetSmallestEntry(ulong stream, long startNumber, long endNumber, out IndexEntry entry) { Ensure.Nonnegative(startNumber, "startNumber"); Ensure.Nonnegative(endNumber, "endNumber"); entry = TableIndex.InvalidIndexEntry; var startKey = BuildKey(stream, startNumber); var endKey = BuildKey(stream, endNumber); if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry)) { return(false); } var workItem = GetWorkItem(); try { IndexEntryKey lowBoundsCheck, highBoundsCheck; var recordRange = LocateRecordRange(startKey, out lowBoundsCheck, out highBoundsCheck); long low = recordRange.Lower; long high = recordRange.Upper; while (low < high) { var mid = low + (high - low + 1) / 2; var midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version); var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version); if (midpointKey.GreaterThan(lowBoundsCheck)) { throw new MaybeCorruptIndexException(String.Format( "Midpoint key (stream: {0}, version: {1}) > low bounds check key (stream: {2}, version: {3})", midpointKey.Stream, midpointKey.Version, lowBoundsCheck.Stream, lowBoundsCheck.Version)); } else if (!midpointKey.GreaterEqualsThan(highBoundsCheck)) { throw new MaybeCorruptIndexException(String.Format( "Midpoint key (stream: {0}, version: {1}) < high bounds check key (stream: {2}, version: {3})", midpointKey.Stream, midpointKey.Version, highBoundsCheck.Stream, highBoundsCheck.Version)); } if (midpointKey.SmallerThan(startKey)) { high = mid - 1; highBoundsCheck = midpointKey; } else { low = mid; lowBoundsCheck = midpointKey; } } var candEntry = ReadEntry(_indexEntrySize, high, workItem, _version); var candidateKey = new IndexEntryKey(candEntry.Stream, candEntry.Version); if (candidateKey.SmallerThan(startKey)) { throw new MaybeCorruptIndexException(string.Format( "candEntry ({0}@{1}) < startKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.", candEntry.Stream, candEntry.Version, startKey, stream, startNumber, endNumber, Filename)); } if (candidateKey.GreaterThan(endKey)) { return(false); } entry = candEntry; return(true); } finally { ReturnWorkItem(workItem); } }
public IEnumerable <IndexEntry> GetRange(ulong stream, long startNumber, long endNumber, int?limit = null) { Ensure.Nonnegative(startNumber, "startNumber"); Ensure.Nonnegative(endNumber, "endNumber"); ulong hash = GetHash(stream); var result = new List <IndexEntry>(); var startKey = BuildKey(hash, startNumber); var endKey = BuildKey(hash, endNumber); if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry)) { return(result); } var workItem = GetWorkItem(); try { IndexEntryKey lowBoundsCheck, highBoundsCheck; var recordRange = LocateRecordRange(endKey, out lowBoundsCheck, out highBoundsCheck); long low = recordRange.Lower; long high = recordRange.Upper; while (low < high) { var mid = low + (high - low) / 2; IndexEntry midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version); var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version); if (midpointKey.GreaterThan(lowBoundsCheck)) { throw new MaybeCorruptIndexException(String.Format( "Midpoint key (stream: {0}, version: {1}) > low bounds check key (stream: {2}, version: {3})", midpointKey.Stream, midpointKey.Version, lowBoundsCheck.Stream, lowBoundsCheck.Version)); } else if (!midpointKey.GreaterEqualsThan(highBoundsCheck)) { throw new MaybeCorruptIndexException(String.Format( "Midpoint key (stream: {0}, version: {1}) < high bounds check key (stream: {2}, version: {3})", midpointKey.Stream, midpointKey.Version, highBoundsCheck.Stream, highBoundsCheck.Version)); } if (midpointKey.SmallerEqualsThan(endKey)) { high = mid; highBoundsCheck = midpointKey; } else { low = mid + 1; lowBoundsCheck = midpointKey; } } PositionAtEntry(_indexEntrySize, high, workItem); for (long i = high, n = Count; i < n; ++i) { IndexEntry entry = ReadNextNoSeek(workItem, _version); var candidateKey = new IndexEntryKey(entry.Stream, entry.Version); if (candidateKey.GreaterThan(endKey)) { throw new MaybeCorruptIndexException(string.Format( "entry ({0}@{1}) > endKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.", entry.Stream, entry.Version, startKey, stream, startNumber, endNumber, Filename)); } if (candidateKey.SmallerThan(startKey)) { return(result); } result.Add(entry); if (result.Count == limit) { break; } } return(result); } finally { ReturnWorkItem(workItem); } }
private IEnumerable <IndexEntry> GetRangeInternal(ulong hash, long startVersion, long endVersion, int?limit = null) { if (startVersion < 0) { throw new ArgumentOutOfRangeException("startVersion"); } if (endVersion < 0) { throw new ArgumentOutOfRangeException("endVersion"); } var candidates = new List <IEnumerator <IndexEntry> >(); var awaiting = _awaitingMemTables; for (int index = 0; index < awaiting.Count; index++) { var range = awaiting[index].Table.GetRange(hash, startVersion, endVersion, limit).GetEnumerator(); if (range.MoveNext()) { candidates.Add(range); } } var map = _indexMap; foreach (var table in map.InOrder()) { var range = table.GetRange(hash, startVersion, endVersion, limit).GetEnumerator(); if (range.MoveNext()) { candidates.Add(range); } } var last = new IndexEntry(0, 0, 0); var first = true; var sortedCandidates = new List <IndexEntry>(); while (candidates.Count > 0) { var maxIdx = GetMaxOf(candidates); var winner = candidates[maxIdx]; var best = winner.Current; if (first || ((last.Stream != best.Stream) && (last.Version != best.Version)) || last.Position != best.Position) { last = best; sortedCandidates.Add(best); first = false; } if (!winner.MoveNext()) { candidates.RemoveAt(maxIdx); } } return(sortedCandidates); }
public bool TryGetOldestEntry(ulong stream, out IndexEntry entry) { ulong hash = GetHash(stream); return(TryGetSmallestEntry(hash, 0, long.MaxValue, out entry)); }
private static PTable MergeTo2(IList <PTable> tables, long numIndexEntries, int indexEntrySize, string outputFile, Func <string, ulong, ulong> upgradeHash, Func <IndexEntry, bool> existsAt, Func <IndexEntry, Tuple <string, bool> > readRecord, byte version, int initialReaders, int maxReaders, int cacheDepth, bool skipIndexVerify) { Log.Debug("PTables merge started (specialized for <= 2 tables)."); var watch = Stopwatch.StartNew(); var fileSizeUpToIndexEntries = GetFileSizeUpToIndexEntries(numIndexEntries, version); var enumerators = tables .Select(table => new EnumerableTable(version, table, upgradeHash, existsAt, readRecord)).ToList(); try { long dumpedEntryCount = 0; using (var f = new FileStream(outputFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, DefaultSequentialBufferSize, FileOptions.SequentialScan)) { f.SetLength(fileSizeUpToIndexEntries); f.Seek(0, SeekOrigin.Begin); using (var md5 = MD5.Create()) using (var cs = new CryptoStream(f, md5, CryptoStreamMode.Write)) using (var bs = new BufferedStream(cs, DefaultSequentialBufferSize)) { // WRITE HEADER var headerBytes = new PTableHeader(version).AsByteArray(); cs.Write(headerBytes, 0, headerBytes.Length); // WRITE INDEX ENTRIES var buffer = new byte[indexEntrySize]; long indexEntry = 0L; List <Midpoint> midpoints = new List <Midpoint>(); var requiredMidpointCount = GetRequiredMidpointCountCached(numIndexEntries, version, cacheDepth); var enum1 = enumerators[0]; var enum2 = enumerators[1]; bool available1 = enum1.MoveNext(); bool available2 = enum2.MoveNext(); IndexEntry current; while (available1 || available2) { var entry1 = new IndexEntry(enum1.Current.Stream, enum1.Current.Version, enum1.Current.Position); var entry2 = new IndexEntry(enum2.Current.Stream, enum2.Current.Version, enum2.Current.Position); if (available1 && (!available2 || entry1.CompareTo(entry2) > 0)) { current = entry1; available1 = enum1.MoveNext(); } else { current = entry2; available2 = enum2.MoveNext(); } AppendRecordTo(bs, buffer, version, current, indexEntrySize); if (version >= PTableVersions.IndexV4 && IsMidpointIndex(indexEntry, numIndexEntries, requiredMidpointCount)) { midpoints.Add(new Midpoint(new IndexEntryKey(current.Stream, current.Version), indexEntry)); } indexEntry++; dumpedEntryCount++; } //WRITE MIDPOINTS if (version >= PTableVersions.IndexV4) { if (dumpedEntryCount != numIndexEntries) { //if index entries have been removed, compute the midpoints again numIndexEntries = dumpedEntryCount; requiredMidpointCount = GetRequiredMidpointCount(numIndexEntries, version, cacheDepth); midpoints = ComputeMidpoints(bs, f, version, indexEntrySize, numIndexEntries, requiredMidpointCount, midpoints); } WriteMidpointsTo(bs, f, version, indexEntrySize, buffer, dumpedEntryCount, numIndexEntries, requiredMidpointCount, midpoints); } bs.Flush(); cs.FlushFinalBlock(); f.SetLength(f.Position + MD5Size); // WRITE MD5 var hash = md5.Hash; f.Write(hash, 0, hash.Length); f.FlushToDisk(); } } Log.Debug( "PTables merge finished in {elapsed} ([{entryCount}] entries merged into {dumpedEntryCount}).", watch.Elapsed, string.Join(", ", tables.Select(x => x.Count)), dumpedEntryCount); return(new PTable(outputFile, Guid.NewGuid(), initialReaders, maxReaders, cacheDepth, skipIndexVerify)); } finally { foreach (var enumerator in enumerators) { enumerator.Dispose(); } } }
private static void AppendRecordTo(Stream stream, byte[] buffer, byte version, IndexEntry entry, int indexEntrySize) { var bytes = entry.Bytes; if (version == PTableVersions.IndexV1) { var entryV1 = new IndexEntryV1((uint)entry.Stream, (int)entry.Version, entry.Position); bytes = entryV1.Bytes; } else if (version == PTableVersions.IndexV2) { var entryV2 = new IndexEntryV2(entry.Stream, (int)entry.Version, entry.Position); bytes = entryV2.Bytes; } Marshal.Copy((IntPtr)bytes, buffer, 0, indexEntrySize); stream.Write(buffer, 0, indexEntrySize); }
private static PTable MergeTo2(IList <PTable> tables, long fileSize, int indexEntrySize, string outputFile, Func <string, ulong, ulong> upgradeHash, Func <IndexEntry, bool> existsAt, Func <IndexEntry, Tuple <string, bool> > readRecord, byte version, int cacheDepth) { Log.Trace("PTables merge started (specialized for <= 2 tables)."); var watch = Stopwatch.StartNew(); var enumerators = tables.Select(table => new EnumerableTable(version, table, upgradeHash, existsAt, readRecord)).ToList(); long dumpedEntryCount = 0; using (var f = new FileStream(outputFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, DefaultSequentialBufferSize, FileOptions.SequentialScan)) { f.SetLength(fileSize); f.Seek(0, SeekOrigin.Begin); using (var md5 = MD5.Create()) using (var cs = new CryptoStream(f, md5, CryptoStreamMode.Write)) using (var bs = new BufferedStream(cs, DefaultSequentialBufferSize)) { // WRITE HEADER var headerBytes = new PTableHeader(version).AsByteArray(); cs.Write(headerBytes, 0, headerBytes.Length); // WRITE INDEX ENTRIES var buffer = new byte[indexEntrySize]; var enum1 = enumerators[0]; var enum2 = enumerators[1]; bool available1 = enum1.MoveNext(); bool available2 = enum2.MoveNext(); IndexEntry current; while (available1 || available2) { var entry1 = new IndexEntry(enum1.Current.Stream, enum1.Current.Version, enum1.Current.Position); var entry2 = new IndexEntry(enum2.Current.Stream, enum2.Current.Version, enum2.Current.Position); if (available1 && (!available2 || entry1.CompareTo(entry2) > 0)) { current = entry1; available1 = enum1.MoveNext(); } else { current = entry2; available2 = enum2.MoveNext(); } if (existsAt(current)) { AppendRecordTo(bs, buffer, version, current, indexEntrySize); dumpedEntryCount += 1; } } bs.Flush(); cs.FlushFinalBlock(); f.SetLength(f.Position + MD5Size); // WRITE MD5 var hash = md5.Hash; f.Write(hash, 0, hash.Length); f.FlushToDisk(); } } Log.Trace("PTables merge finished in {0} ([{1}] entries merged into {2}).", watch.Elapsed, string.Join(", ", tables.Select(x => x.Count)), dumpedEntryCount); return(new PTable(outputFile, Guid.NewGuid(), version, depth: cacheDepth)); }
private bool TryGetSmallestEntry(uint stream, int startNumber, int endNumber, out IndexEntry entry) { Ensure.Nonnegative(startNumber, "startNumber"); Ensure.Nonnegative(endNumber, "endNumber"); entry = TableIndex.InvalidIndexEntry; var startKey = BuildKey(stream, startNumber); var endKey = BuildKey(stream, endNumber); if (startKey > _maxEntry || endKey < _minEntry) { return(false); } var workItem = GetWorkItem(); try { var recordRange = LocateRecordRange(startKey); int low = recordRange.Lower; int high = recordRange.Upper; while (low < high) { var mid = low + (high - low + 1) / 2; IndexEntry midpoint = ReadEntry(mid, workItem); if (midpoint.Key < startKey) { high = mid - 1; } else { low = mid; } } var candEntry = ReadEntry(high, workItem); if (candEntry.Key < startKey) { throw new Exception(string.Format("candEntry.Key {0} < startKey {1}, stream {2}, startNum {3}, endNum {4}, PTable: {5}.", candEntry.Key, startKey, stream, startNumber, endNumber, Filename)); } if (candEntry.Key > endKey) { return(false); } entry = candEntry; return(true); } finally { ReturnWorkItem(workItem); } }
public bool TryGetOldestEntry(uint stream, out IndexEntry entry) { return(TryGetSmallestEntry(stream, 0, int.MaxValue, out entry)); }
public bool TryGetLatestEntry(uint stream, out IndexEntry entry) { return(TryGetLargestEntry(stream, 0, int.MaxValue, out entry)); }
private bool TryGetLargestEntry(ulong stream, long startNumber, long endNumber, out IndexEntry entry) { Ensure.Nonnegative(startNumber, "startNumber"); Ensure.Nonnegative(endNumber, "endNumber"); entry = TableIndex.InvalidIndexEntry; var startKey = BuildKey(stream, startNumber); var endKey = BuildKey(stream, endNumber); if (startKey.GreaterThan(_maxEntry) || endKey.SmallerThan(_minEntry)) { return(false); } var workItem = GetWorkItem(); try { var recordRange = LocateRecordRange(endKey); long low = recordRange.Lower; long high = recordRange.Upper; while (low < high) { var mid = low + (high - low) / 2; IndexEntry midpoint = ReadEntry(_indexEntrySize, mid, workItem, _version); var midpointKey = new IndexEntryKey(midpoint.Stream, midpoint.Version); if (midpointKey.GreaterThan(endKey)) { low = mid + 1; } else { high = mid; } } var candEntry = ReadEntry(_indexEntrySize, high, workItem, _version); var candKey = new IndexEntryKey(candEntry.Stream, candEntry.Version); if (candKey.GreaterThan(endKey)) { throw new Exception(string.Format("candEntry ({0}@{1}) > startKey {2}, stream {3}, startNum {4}, endNum {5}, PTable: {6}.", candEntry.Stream, candEntry.Version, startKey, stream, startNumber, endNumber, Filename)); } if (candKey.SmallerThan(startKey)) { return(false); } entry = candEntry; return(true); } finally { ReturnWorkItem(workItem); } }