public IOScheduler(int maxSkiplistBacklog, UnsortedStorageLayer storageLayer, TrimDatabase database)
{
_storageLayer = storageLayer;
_database = database;
_sortedStrategy = (sl) => false;
_unsortedStrategy = (sl) => sl.NumberOfTables > (sl.MaxFilesAtLayer * 0.8);
_channel = Channel.CreateBounded <MemoryTable>(new BoundedChannelOptions(maxSkiplistBacklog));
_writerTask = WriteInMemoryTable();
_mergeTask = CheckForMerge();
}
private async Task MergeUnsortedLayer(UnsortedStorageLayer unsortedLayer)
{
// We can't merge downwards at the bottom layer
if (unsortedLayer.Level == _database.StorageLayers.Count)
{
return;
}
var nextLayer = _database.StorageLayers[unsortedLayer.Level];
var tables = unsortedLayer.GetTables();
var oldestTable = tables[0];
// Get all of the overlapping tables
var overlapped = GetOverlappingTables(oldestTable, nextLayer);
overlapped.Insert(0, oldestTable);
Console.WriteLine($"Begin merging with {overlapped.Count} tables");
var sw = Stopwatch.StartNew();
await using (var merger = new TableFileMerger(overlapped.Select(ol => ol.GetAsyncEnumerator()).ToArray()))
{
// Begin writing out to disk
var writer = new TableFileMergeWriter(_database, nextLayer, _database.BlockCache, unsortedLayer.Level);
await writer.WriteFromMerger(merger);
nextLayer.AddAndRemoveTableFiles(writer.NewTableFiles, overlapped);
unsortedLayer.RemoveTable(oldestTable);
}
foreach (var file in overlapped)
{
file.Dispose();
System.IO.File.Delete(file.FileName);
}
Console.WriteLine($"Finished merging in {sw.ElapsedMilliseconds}ms");
}
