public void Truncate()
{
_currentJournaledMemTable.Close();
TableManager.Default.Close(this);
foreach (var pair in _secondaryIndexes)
{
pair.Value.Close(FastClose);
}
string basePath = Path.GetFullPath(Manifest.BaseFileName);
foreach (string file in Directory.GetFiles(basePath, "*.*", SearchOption.AllDirectories))
{
Helper.DeleteFile(file, false, (msg) => { Manifest.LogMessage(msg); });
}
foreach (string dir in Directory.GetDirectories(basePath, "*.*", SearchOption.AllDirectories))
{
Helper.DeleteFolder(dir, false, (msg) => { Manifest.LogMessage(msg); });
}
_manifest = new Manifest(basePath);
_currentJournaledMemTable = new JournaledMemTable(_manifest.BaseFileName, _manifest.CurrentVersion(0));
_cache = new RazorCache();
_secondaryIndexes = new Dictionary <string, KeyValueStore>(StringComparer.OrdinalIgnoreCase);
Manifest.LogMessage("Database Truncated.");
}
public static IEnumerable <KeyValuePair <Key, Value> > Enumerate(int level, RazorCache rzrCache, string baseFileName, ManifestImmutable mft, Key key)
{
var enumerator = new TableEnumerator(level, rzrCache, baseFileName, mft, key);
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
public SortedBlockTable(RazorCache cache, string baseFileName, int level, int version)
{
PerformanceCounters.SBTConstructed.Increment();
_baseFileName = baseFileName;
_level = level;
_version = version;
_cache = cache;
_path = Config.SortedBlockTableFile(baseFileName, level, version);
ReadMetadata();
}
public static IEnumerable <PageRecord> MergeTables(RazorCache cache, Manifest mf, int destinationLevel, IEnumerable <PageRef> tableSpecs, ExceptionHandling exceptionHandling, Action <string> logger)
{
var orderedTableSpecs = tableSpecs.OrderByPagePriority();
var outputTables = new List <PageRecord>();
SortedBlockTableWriter writer = null;
Key firstKey = new Key();
Key lastKey = new Key();
Key maxKey = new Key(); // Maximum key we can span with this table to avoid covering more than 10 pages in the destination
Action <KeyValuePair <Key, Value> > OpenPage = (pair) => {
writer = new SortedBlockTableWriter(mf.BaseFileName, destinationLevel, mf.NextVersion(destinationLevel));
firstKey = pair.Key;
using (var m = mf.GetLatestManifest()) {
maxKey = m.FindSpanningLimit(destinationLevel + 1, firstKey);
}
};
Action ClosePage = () => {
writer.Close();
outputTables.Add(new PageRecord(destinationLevel, writer.Version, firstKey, lastKey));
writer = null;
};
try {
foreach (var pair in EnumerateMergedTablesPreCached(cache, mf.BaseFileName, orderedTableSpecs, exceptionHandling, logger))
{
if (writer == null)
{
OpenPage(pair);
}
if (writer.WrittenSize >= Config.MaxSortedBlockTableSize || (!maxKey.IsEmpty && pair.Key.CompareTo(maxKey) >= 0))
{
ClosePage();
}
if (writer == null)
{
OpenPage(pair);
}
writer.WritePair(pair.Key, pair.Value);
lastKey = pair.Key;
}
} finally {
if (writer != null)
{
ClosePage();
}
}
return(outputTables);
}
public KeyValueStore(string baseFileName, RazorCache cache)
{
if (!Directory.Exists(baseFileName))
{
Directory.CreateDirectory(baseFileName);
}
_manifest = new Manifest(baseFileName);
_manifest.Logger = Config.Logger;
int memTableVersion = _manifest.CurrentVersion(0);
// Check for a previously aborted journal rotation
CheckForIncompleteJournalRotation(baseFileName, memTableVersion);
// Create new journal for this run (and potentially load from disk, if there was data loaded previously)
_currentJournaledMemTable = new JournaledMemTable(_manifest.BaseFileName, memTableVersion);
_cache = cache == null ? new RazorCache() : cache;
}
public static bool Lookup(string baseFileName, int level, int version, RazorCache cache, Key key, out Value value, ExceptionHandling exceptionHandling, Action <string> logger)
{
PerformanceCounters.SBTLookup.Increment();
SortedBlockTable sbt = new SortedBlockTable(cache, baseFileName, level, version);
try {
byte[] lastScanKey;
int dataBlockNum = FindBlockForKey(baseFileName, level, version, cache, key, out lastScanKey);
if (dataBlockNum >= 0 && dataBlockNum < sbt._dataBlocks)
{
byte[] block = sbt.ReadBlock(LocalThreadAllocatedBlock(), dataBlockNum, lastScanKey);
return(sbt.ScanBlockForKey(block, key, out value));
}
} finally {
sbt.Close();
}
value = Value.Empty;
return(false);
}
public TableEnumerator(int level, RazorCache rzrCache, string baseFileName, ManifestImmutable mft, Key key)
{
Cache = rzrCache;
BaseFileName = baseFileName;
ActiveManifest = mft;
StartKey = key;
Level = level;
StopEnumerating = false;
var firstPage = ActiveManifest.FindPageForIndex(level, 0);
if (firstPage != null && key.CompareTo(firstPage.FirstKey) < 0)
{
StartPageIndex = 0;
}
else
{
StartPageIndex = ActiveManifest.FindPageIndexForKey(Level, StartKey);
}
InitEnumerator();
}
public static void RunTableMergePass(KeyValueStore kvStore)
{
try {
Interlocked.Increment(ref kvStore.mergeCount);
lock (kvStore.mergeLock) {
RazorCache cache = kvStore.Cache;
Manifest manifest = kvStore.Manifest;
while (true)
{
bool mergedDuringLastPass = false;
using (var manifestInst = kvStore.Manifest.GetLatestManifest()) {
// Handle level 0 (merge all pages)
if (manifestInst.GetNumPagesAtLevel(0) >= Config.MaxPagesOnLevel(0))
{
mergedDuringLastPass = true;
var inputPageRecords = manifestInst.GetPagesAtLevel(0).OrderBy(p => p.Version).ToList();
var startKey = inputPageRecords.Min(p => p.FirstKey);
var endKey = inputPageRecords.Max(p => p.LastKey);
var mergePages = manifestInst.FindPagesForKeyRange(1, startKey, endKey).AsPageRefs().ToList();
var allInputPages = inputPageRecords.AsPageRefs().Concat(mergePages).ToList();
var outputPages = SortedBlockTable.MergeTables(cache, manifest, 1, allInputPages, ExceptionHandling.ThrowAll, null).ToList();
manifest.ModifyPages(outputPages, allInputPages);
manifest.LogMessage("Merge Level 0 => InputPages: {0} OutputPages:{1}",
string.Join(",", allInputPages.Select(p => string.Format("{0}-{1}", p.Level, p.Version)).ToArray()),
string.Join(",", outputPages.Select(p => string.Format("{0}-{1}", p.Level, p.Version)).ToArray())
);
}
// handle the rest of the levels (merge only one page upwards)
for (int level = 1; level < manifestInst.NumLevels - 1; level++)
{
if (manifestInst.GetNumPagesAtLevel(level) >= Config.MaxPagesOnLevel(level))
{
mergedDuringLastPass = true;
var inputPage = manifest.NextMergePage(level);
var mergePages = manifestInst.FindPagesForKeyRange(level + 1, inputPage.FirstKey, inputPage.LastKey).ToList();
var inputPageRecords = mergePages.Concat(new PageRecord[] { inputPage });
var allInputPages = inputPageRecords.AsPageRefs().ToList();
var outputPages = SortedBlockTable.MergeTables(cache, manifest, level + 1, allInputPages, ExceptionHandling.ThrowAll, null);
// Notify if a merge happened, implemented for testing primarily
if (kvStore.MergeCallback != null)
{
kvStore.MergeCallback(level, inputPageRecords, outputPages);
}
manifest.ModifyPages(outputPages, allInputPages);
manifest.LogMessage("Merge Level >0 => InputPages: {0} OutputPages:{1}",
string.Join(",", allInputPages.Select(p => string.Format("{0}-{1}", p.Level, p.Version)).ToArray()),
string.Join(",", outputPages.Select(p => string.Format("{0}-{1}", p.Level, p.Version)).ToArray())
);
}
}
}
// No more merging is needed, we are finished with this pass
if (!mergedDuringLastPass)
{
return;
}
}
}
} finally {
Interlocked.Decrement(ref kvStore.mergeCount);
}
}
public static IEnumerable <KeyValuePair <Key, Value> > EnumerateMergedTablesPreCached(RazorCache cache, string baseFileName, IEnumerable <PageRef> tableSpecs, ExceptionHandling exceptionHandling, Action <string> logger)
{
PerformanceCounters.SBTEnumerateMergedTablesPrecached.Increment();
var tables = tableSpecs
.Select(pageRef => new SortedBlockTable(cache, baseFileName, pageRef.Level, pageRef.Version))
.ToList();
try {
foreach (var pair in MergeEnumerator.Merge(tables.Select(t => t.Enumerate().ToList().AsEnumerable()), t => t.Key))
{
yield return(pair);
}
} finally {
tables.ForEach(t => t.Close());
}
}
public IEnumerable <RawRecord> EnumerateFromKeyRaw(RazorCache indexCache, Key key)
{
if (!FileExists)
{
yield break;
}
int startingBlock;
if (key.Length == 0)
{
startingBlock = 0;
}
else
{
startingBlock = FindBlockForKey(_baseFileName, _level, _version, indexCache, key, out _lastScanKey);
if (startingBlock < 0)
{
startingBlock = 0;
}
}
if (startingBlock < _dataBlocks)
{
byte[] allocBlockA = new byte[Config.SortedBlockSize];
byte[] allocBlockB = new byte[Config.SortedBlockSize];
byte[] currentBlock = allocBlockA;
var asyncResult = BeginReadBlock(currentBlock, startingBlock);
try {
for (int i = startingBlock; i < _dataBlocks; i++)
{
// wait on last block read to complete so we can start processing the data
byte[] block = EndReadBlock(asyncResult);
asyncResult = null;
// Go ahead and kick off the next block read asynchronously while we parse the last one
if (i < _dataBlocks)
{
SwapBlocks(allocBlockA, allocBlockB, ref currentBlock); // swap the blocks so we can issue another disk i/o
asyncResult = BeginReadBlock(currentBlock, i + 1);
}
int offset = FormatVersion < 2 ? 2 : 02; // handle old format with 2 bytes for offset to treehead
// On the first block, we need to seek to the key first (if we don't have an empty key)
if (i == startingBlock && key.Length != 0)
{
while (offset >= 0)
{
var rec = ReadRawRecord(ref block, ref offset);
if (rec.Key.CompareTo(key) >= 0)
{
yield return(rec);
break;
}
}
}
// Now loop through the rest of the block
while (offset >= 0)
{
yield return(ReadRawRecord(ref block, ref offset));
}
}
} finally {
if (asyncResult != null)
{
EndReadBlock(asyncResult);
}
}
}
}
private static int FindBlockForKey(string baseFileName, int level, int version, RazorCache indexCache, Key key, out byte[] baseKey)
{
Key[] index = indexCache.GetBlockTableIndex(baseFileName, level, version);
int dataBlockNum = Array.BinarySearch(index, key);
if (dataBlockNum < 0)
{
dataBlockNum = ~dataBlockNum - 1;
}
baseKey = (dataBlockNum > -1 && dataBlockNum < index.Length) ? index[dataBlockNum].InternalBytes : Key.Empty.InternalBytes;
return(dataBlockNum);
}
