private async Task MoveTableToLowerLevel(SortedStorageLayer sortedStorage, StorageLayer layerBelow, TableFile file)
{
var newFilename = layerBelow.GetNextFileName();
System.IO.File.Copy(file.FileName, newFilename);
var tableFile = new TableFile(newFilename, _database.BlockCache);
await tableFile.LoadAsync();
layerBelow.AddTableFile(tableFile);
sortedStorage.RemoveTable(file);
file.Dispose();
System.IO.File.Delete(file.FileName);
Console.WriteLine("Moved table down with no merge");
// Move table down one level
}
private async Task MergeSortedLayer(SortedStorageLayer sortedStorage)
{
var layerBelow = _database.StorageLayers[sortedStorage.Level];
if (layerBelow.NumberOfTables == 0)
{
var file = sortedStorage.GetTables()[0];
await MoveTableToLowerLevel(sortedStorage, layerBelow, file);
return;
}
var overlapCounts = new int[sortedStorage.NumberOfTables];
for (var i = 0; i < sortedStorage.GetTables().Length; i++)
{
var t = sortedStorage.GetTables()[i];
// Check if there is overlap
var overlapCount = 0;
foreach (var l3t in layerBelow.GetTables())
{
if (t.LastKey.Span.SequenceCompareTo(l3t.FirstKey.Span) < 0)
{
continue;
}
if (t.FirstKey.Span.SequenceCompareTo(l3t.LastKey.Span) > 0)
{
continue;
}
overlapCount++;
}
if (overlapCount == 0)
{
// Move table down one level
await MoveTableToLowerLevel(sortedStorage, layerBelow, t);
return;
}
overlapCounts[i] = overlapCount;
}
var min = overlapCounts.Min();
var indexOfMin = overlapCounts.Select((value, index) => (value, index)).First(i => i.value == min).index;
var upperTable = sortedStorage.GetTables()[indexOfMin];
var overlapping = GetOverlappingTables(upperTable, layerBelow);
overlapping.Insert(0, upperTable);
Console.WriteLine($"Merging from {sortedStorage.Level}");
var sw = Stopwatch.StartNew();
await using (var merger = new TableFileMerger(overlapping.Select(ol => ol.GetAsyncEnumerator()).ToArray()))
{
// Begin writing out to disk
var writer = new TableFileMergeWriter(layerBelow, _database.BlockCache);
await writer.WriteFromMerger(merger);
layerBelow.AddAndRemoveTableFiles(writer.NewTableFiles, overlapping);
sortedStorage.RemoveTable(upperTable);
}
Console.WriteLine($"Merge for level {sortedStorage.Level} took {sw.ElapsedMilliseconds}ms");
foreach (var file in overlapping)
{
file.Dispose();
System.IO.File.Delete(file.FileName);
}
// Found with min overlap so merge it
return;
}
