private IEnumerator <KeyValuePair <Key, Value> > GetEnumerator()
{
if (NextPageIndex < 0)
{
StopEnumerating = true;
yield break;
}
PageRecord page = ActiveManifest.FindPageForIndex(Level, NextPageIndex++);
if (page == null)
{
StopEnumerating = true;
yield break;
}
var sbt = new SortedBlockTable(Cache, BaseFileName, page.Level, page.Version);
try {
foreach (var pair in sbt.EnumerateFromKey(Cache, StartKey))
{
yield return(pair);
}
} finally {
sbt.Close();
}
}
private Value InternalGet(Key lookupKey)
{
Value output = Value.Empty;
// Capture copy of the rotated table if there is one
var rotatedMemTable = _rotatedJournaledMemTable;
// somtimes on shutdown this is null
if (_currentJournaledMemTable == null || _manifest == null)
{
return(Value.Empty);
}
// First check the current memtable
if (_currentJournaledMemTable.Lookup(lookupKey, out output))
{
return(output);
}
// Check the table in rotation
if (rotatedMemTable != null)
{
if (rotatedMemTable.Lookup(lookupKey, out output))
{
return(output);
}
}
// Now check the files on disk
using (var manifest = _manifest.GetLatestManifest()) {
// Must check all pages on level 0
var zeroPages = manifest.GetPagesAtLevel(0).OrderByDescending((page) => page.Version);
foreach (var page in zeroPages)
{
if (SortedBlockTable.Lookup(_manifest.BaseFileName, page.Level, page.Version, _cache, lookupKey, out output, _exceptionHandling, _manifest.Logger))
{
return(output);
}
}
// If not found, must check pages on the higher levels, but we can use the page index to make the search quicker
for (int level = 1; level < manifest.NumLevels; level++)
{
var page = manifest.FindPageForKey(level, lookupKey);
if (page != null && SortedBlockTable.Lookup(_manifest.BaseFileName, page.Level, page.Version, _cache, lookupKey, out output, _exceptionHandling, _manifest.Logger))
{
return(output);
}
}
}
// OK, not found anywhere, return null
return(Value.Empty);
}
public static bool Lookup(string baseFileName, int level, int version, RazorCache cache, Key key, out Value value, ExceptionHandling exceptionHandling, Action <string> logger)
{
PerformanceCounters.SBTLookup.Increment();
SortedBlockTable sbt = new SortedBlockTable(cache, baseFileName, level, version);
try {
byte[] lastScanKey;
int dataBlockNum = FindBlockForKey(baseFileName, level, version, cache, key, out lastScanKey);
if (dataBlockNum >= 0 && dataBlockNum < sbt._dataBlocks)
{
byte[] block = sbt.ReadBlock(LocalThreadAllocatedBlock(), dataBlockNum, lastScanKey);
return(sbt.ScanBlockForKey(block, key, out value));
}
} finally {
sbt.Close();
}
value = Value.Empty;
return(false);
}
public Key[] GetBlockTableIndex(string baseName, int level, int version)
{
string fileName = Config.SortedBlockTableFile(baseName, level, version);
Key[] index;
if (_blockIndexCache.TryGetValue(fileName, out index))
{
return(index);
}
PerformanceCounters.SBTGetBlockTableIndex.Increment();
var sbt = new SortedBlockTable(null, baseName, level, version);
try {
index = sbt.GetIndex();
_blockIndexCache.Set(fileName, index);
return(index);
} finally {
sbt.Close();
}
}
public static void RunTableMergePass(KeyValueStore kvStore)
{
try {
Interlocked.Increment(ref kvStore.mergeCount);
lock (kvStore.mergeLock) {
RazorCache cache = kvStore.Cache;
Manifest manifest = kvStore.Manifest;
while (true)
{
bool mergedDuringLastPass = false;
using (var manifestInst = kvStore.Manifest.GetLatestManifest()) {
// Handle level 0 (merge all pages)
if (manifestInst.GetNumPagesAtLevel(0) >= Config.MaxPagesOnLevel(0))
{
mergedDuringLastPass = true;
var inputPageRecords = manifestInst.GetPagesAtLevel(0).OrderBy(p => p.Version).ToList();
var startKey = inputPageRecords.Min(p => p.FirstKey);
var endKey = inputPageRecords.Max(p => p.LastKey);
var mergePages = manifestInst.FindPagesForKeyRange(1, startKey, endKey).AsPageRefs().ToList();
var allInputPages = inputPageRecords.AsPageRefs().Concat(mergePages).ToList();
var outputPages = SortedBlockTable.MergeTables(cache, manifest, 1, allInputPages, ExceptionHandling.ThrowAll, null).ToList();
manifest.ModifyPages(outputPages, allInputPages);
manifest.LogMessage("Merge Level 0 => InputPages: {0} OutputPages:{1}",
string.Join(",", allInputPages.Select(p => string.Format("{0}-{1}", p.Level, p.Version)).ToArray()),
string.Join(",", outputPages.Select(p => string.Format("{0}-{1}", p.Level, p.Version)).ToArray())
);
}
// handle the rest of the levels (merge only one page upwards)
for (int level = 1; level < manifestInst.NumLevels - 1; level++)
{
if (manifestInst.GetNumPagesAtLevel(level) >= Config.MaxPagesOnLevel(level))
{
mergedDuringLastPass = true;
var inputPage = manifest.NextMergePage(level);
var mergePages = manifestInst.FindPagesForKeyRange(level + 1, inputPage.FirstKey, inputPage.LastKey).ToList();
var inputPageRecords = mergePages.Concat(new PageRecord[] { inputPage });
var allInputPages = inputPageRecords.AsPageRefs().ToList();
var outputPages = SortedBlockTable.MergeTables(cache, manifest, level + 1, allInputPages, ExceptionHandling.ThrowAll, null);
// Notify if a merge happened, implemented for testing primarily
if (kvStore.MergeCallback != null)
{
kvStore.MergeCallback(level, inputPageRecords, outputPages);
}
manifest.ModifyPages(outputPages, allInputPages);
manifest.LogMessage("Merge Level >0 => InputPages: {0} OutputPages:{1}",
string.Join(",", allInputPages.Select(p => string.Format("{0}-{1}", p.Level, p.Version)).ToArray()),
string.Join(",", outputPages.Select(p => string.Format("{0}-{1}", p.Level, p.Version)).ToArray())
);
}
}
}
// No more merging is needed, we are finished with this pass
if (!mergedDuringLastPass)
{
return;
}
}
}
} finally {
Interlocked.Decrement(ref kvStore.mergeCount);
}
}